Lines Matching refs:DeviceId
444 void addOffloadEntry(const int DeviceId, const __tgt_offload_entry Entry) { in addOffloadEntry() argument
445 FuncOrGblEntryTy &E = DeviceData[DeviceId].FuncGblEntries.back(); in addOffloadEntry()
450 const __tgt_offload_entry *getOffloadEntry(const int DeviceId, in getOffloadEntry() argument
453 DeviceData[DeviceId].FuncGblEntries.back().Entries) in getOffloadEntry()
461 __tgt_target_table *getOffloadEntriesTable(const int DeviceId) { in getOffloadEntriesTable() argument
462 FuncOrGblEntryTy &E = DeviceData[DeviceId].FuncGblEntries.back(); in getOffloadEntriesTable()
475 void clearOffloadEntriesTable(const int DeviceId) { in clearOffloadEntriesTable() argument
476 DeviceData[DeviceId].FuncGblEntries.emplace_back(); in clearOffloadEntriesTable()
477 FuncOrGblEntryTy &E = DeviceData[DeviceId].FuncGblEntries.back(); in clearOffloadEntriesTable()
483 CUstream getStream(const int DeviceId, __tgt_async_info *AsyncInfo) const { in getStream() argument
488 if (StreamPool[DeviceId]->acquire(S) != OFFLOAD_SUCCESS) in getStream()
585 for (int DeviceId = 0; DeviceId < NumberOfDevices; ++DeviceId) in ~DeviceRTLTy() local
586 deinitDevice(DeviceId); in ~DeviceRTLTy()
590 bool isValidDeviceId(const int DeviceId) const { in isValidDeviceId()
591 return DeviceId >= 0 && DeviceId < NumberOfDevices; in isValidDeviceId()
598 int initDevice(const int DeviceId) { in initDevice() argument
601 DP("Getting device %d\n", DeviceId); in initDevice()
602 CUresult Err = cuDeviceGet(&Device, DeviceId); in initDevice()
606 assert(InitializedFlags[DeviceId] == false && "Reinitializing device!"); in initDevice()
607 InitializedFlags[DeviceId] = true; in initDevice()
633 Err = cuDevicePrimaryCtxRetain(&DeviceData[DeviceId].Context, Device); in initDevice()
637 Err = cuCtxSetCurrent(DeviceData[DeviceId].Context); in initDevice()
642 if (!StreamPool[DeviceId]) in initDevice()
643 StreamPool[DeviceId] = std::make_unique<StreamPoolTy>(StreamAllocatorTy(), in initDevice()
647 if (!EventPool[DeviceId]) in initDevice()
648 EventPool[DeviceId] = in initDevice()
658 DeviceData[DeviceId].BlocksPerGrid = DeviceRTLTy::DefaultNumTeams; in initDevice()
661 DeviceData[DeviceId].BlocksPerGrid = MaxGridDimX; in initDevice()
671 DeviceData[DeviceId].ThreadsPerBlock = DeviceRTLTy::DefaultNumThreads; in initDevice()
674 DeviceData[DeviceId].ThreadsPerBlock = MaxBlockDimX; in initDevice()
677 DeviceData[DeviceId].ThreadsPerBlock > EnvTeamThreadLimit) { in initDevice()
680 DeviceData[DeviceId].ThreadsPerBlock, EnvTeamThreadLimit); in initDevice()
681 DeviceData[DeviceId].ThreadsPerBlock = EnvTeamThreadLimit; in initDevice()
683 if (DeviceData[DeviceId].ThreadsPerBlock > DeviceRTLTy::HardThreadLimit) { in initDevice()
686 DeviceData[DeviceId].ThreadsPerBlock, DeviceRTLTy::HardThreadLimit); in initDevice()
687 DeviceData[DeviceId].ThreadsPerBlock = DeviceRTLTy::HardThreadLimit; in initDevice()
697 DeviceData[DeviceId].WarpSize = 32; in initDevice()
700 DeviceData[DeviceId].WarpSize = WarpSize; in initDevice()
704 if (EnvTeamLimit > 0 && DeviceData[DeviceId].BlocksPerGrid > EnvTeamLimit) { in initDevice()
707 DeviceData[DeviceId].BlocksPerGrid = EnvTeamLimit; in initDevice()
729 INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId, in initDevice()
732 DeviceData[DeviceId].BlocksPerGrid, in initDevice()
733 DeviceData[DeviceId].ThreadsPerBlock, DeviceData[DeviceId].WarpSize); in initDevice()
734 INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId, in initDevice()
743 DeviceData[DeviceId].NumTeams = EnvNumTeams; in initDevice()
745 DeviceData[DeviceId].NumTeams = DeviceRTLTy::DefaultNumTeams; in initDevice()
750 if (DeviceData[DeviceId].NumTeams > DeviceData[DeviceId].BlocksPerGrid) { in initDevice()
752 DeviceData[DeviceId].BlocksPerGrid); in initDevice()
753 DeviceData[DeviceId].NumTeams = DeviceData[DeviceId].BlocksPerGrid; in initDevice()
757 DeviceData[DeviceId].NumThreads = DeviceRTLTy::DefaultNumThreads; in initDevice()
760 if (DeviceData[DeviceId].NumThreads > in initDevice()
761 DeviceData[DeviceId].ThreadsPerBlock) { in initDevice()
763 DeviceData[DeviceId].ThreadsPerBlock); in initDevice()
764 DeviceData[DeviceId].NumThreads = DeviceData[DeviceId].ThreadsPerBlock; in initDevice()
770 int deinitDevice(const int DeviceId) { in deinitDevice() argument
771 auto IsInitialized = InitializedFlags[DeviceId]; in deinitDevice()
774 InitializedFlags[DeviceId] = false; in deinitDevice()
777 MemoryManagers[DeviceId].release(); in deinitDevice()
779 StreamPool[DeviceId].reset(); in deinitDevice()
780 EventPool[DeviceId].reset(); in deinitDevice()
782 DeviceDataTy &D = DeviceData[DeviceId]; in deinitDevice()
788 for (auto &M : Modules[DeviceId]) in deinitDevice()
806 __tgt_target_table *loadBinary(const int DeviceId, in loadBinary() argument
809 clearOffloadEntriesTable(DeviceId); in loadBinary()
821 Modules[DeviceId].push_back(Module); in loadBinary()
827 std::list<KernelTy> &KernelsList = DeviceData[DeviceId].KernelsList; in loadBinary()
879 addOffloadEntry(DeviceId, Entry); in loadBinary()
930 addOffloadEntry(DeviceId, Entry); in loadBinary()
937 static_cast<uint32_t>(DeviceId), in loadBinary()
975 return getOffloadEntriesTable(DeviceId); in loadBinary()
978 void *dataAlloc(const int DeviceId, const int64_t Size, in dataAlloc() argument
984 return MemoryManagers[DeviceId]->allocate(Size, nullptr); in dataAlloc()
986 return DeviceAllocators[DeviceId].allocate(Size, nullptr, Kind); in dataAlloc()
989 return DeviceAllocators[DeviceId].allocate(Size, nullptr, Kind); in dataAlloc()
998 int dataSubmit(const int DeviceId, const void *TgtPtr, const void *HstPtr, in dataSubmit() argument
1002 CUstream Stream = getStream(DeviceId, AsyncInfo); in dataSubmit()
1015 int dataRetrieve(const int DeviceId, void *HstPtr, const void *TgtPtr, in dataRetrieve() argument
1019 CUstream Stream = getStream(DeviceId, AsyncInfo); in dataRetrieve()
1105 int dataDelete(const int DeviceId, void *TgtPtr) { in dataDelete() argument
1107 return MemoryManagers[DeviceId]->free(TgtPtr); in dataDelete()
1109 return DeviceAllocators[DeviceId].free(TgtPtr); in dataDelete()
1112 int runTargetTeamRegion(const int DeviceId, void *TgtEntryPtr, void **TgtArgs, in runTargetTeamRegion() argument
1141 DP("Adding master warp: +%d threads\n", DeviceData[DeviceId].WarpSize); in runTargetTeamRegion()
1142 CudaThreadsPerBlock += DeviceData[DeviceId].WarpSize; in runTargetTeamRegion()
1146 DeviceData[DeviceId].NumThreads); in runTargetTeamRegion()
1147 CudaThreadsPerBlock = DeviceData[DeviceId].NumThreads; in runTargetTeamRegion()
1150 if (CudaThreadsPerBlock > DeviceData[DeviceId].ThreadsPerBlock) { in runTargetTeamRegion()
1152 DeviceData[DeviceId].ThreadsPerBlock); in runTargetTeamRegion()
1153 CudaThreadsPerBlock = DeviceData[DeviceId].ThreadsPerBlock; in runTargetTeamRegion()
1211 DP("Using default number of teams %d\n", DeviceData[DeviceId].NumTeams); in runTargetTeamRegion()
1212 CudaBlocksPerGrid = DeviceData[DeviceId].NumTeams; in runTargetTeamRegion()
1219 if (CudaBlocksPerGrid > DeviceData[DeviceId].BlocksPerGrid) { in runTargetTeamRegion()
1221 DeviceData[DeviceId].BlocksPerGrid); in runTargetTeamRegion()
1222 CudaBlocksPerGrid = DeviceData[DeviceId].BlocksPerGrid; in runTargetTeamRegion()
1225 INFO(OMP_INFOTYPE_PLUGIN_KERNEL, DeviceId, in runTargetTeamRegion()
1227 (getOffloadEntry(DeviceId, TgtEntryPtr)) in runTargetTeamRegion()
1228 ? getOffloadEntry(DeviceId, TgtEntryPtr)->name in runTargetTeamRegion()
1233 CUstream Stream = getStream(DeviceId, AsyncInfo); in runTargetTeamRegion()
1247 int synchronize(const int DeviceId, __tgt_async_info *AsyncInfo) const { in synchronize() argument
1254 StreamPool[DeviceId]->release(reinterpret_cast<CUstream>(AsyncInfo->Queue)); in synchronize()
1266 void printDeviceInfo(int32_t DeviceId) { in printDeviceInfo() argument
1273 checkResult(cuDeviceGet(&Device, DeviceId), in printDeviceInfo()
1278 printf(" CUDA Device Number: \t\t%d \n", DeviceId); in printDeviceInfo()
1440 int createEvent(int DeviceId, void **P) { in createEvent() argument
1442 if (EventPool[DeviceId]->acquire(Event) != OFFLOAD_SUCCESS) in createEvent()
1448 int destroyEvent(int DeviceId, void *EventPtr) { in destroyEvent() argument
1449 EventPool[DeviceId]->release(reinterpret_cast<CUevent>(EventPtr)); in destroyEvent()
1453 int waitEvent(const int DeviceId, __tgt_async_info *AsyncInfo, in waitEvent() argument
1455 CUstream Stream = getStream(DeviceId, AsyncInfo); in waitEvent()
1472 int releaseAsyncInfo(int DeviceId, __tgt_async_info *AsyncInfo) const { in releaseAsyncInfo() argument
1474 StreamPool[DeviceId]->release( in releaseAsyncInfo()
1482 int initAsyncInfo(int DeviceId, __tgt_async_info **AsyncInfo) const { in initAsyncInfo() argument
1484 getStream(DeviceId, *AsyncInfo); in initAsyncInfo()
1488 int initDeviceInfo(int DeviceId, __tgt_device_info *DeviceInfo, in initDeviceInfo() argument
1493 DeviceInfo->Context = DeviceData[DeviceId].Context; in initDeviceInfo()
1496 CUresult Err = cuDeviceGet(&Dev, DeviceId); in initDeviceInfo()
1507 int setContext(int DeviceId) { in setContext() argument
1508 assert(InitializedFlags[DeviceId] && "Device is not initialized"); in setContext()
1510 CUresult Err = cuCtxSetCurrent(DeviceData[DeviceId].Context); in setContext()
1544 for (int32_t DeviceId = 0; DeviceId < NumberOfDevices; ++DeviceId) { in __tgt_rtl_is_valid_binary_info() local
1546 if (cuDeviceGet(&Device, DeviceId) != CUDA_SUCCESS) in __tgt_rtl_is_valid_binary_info()
1587 int32_t __tgt_rtl_init_device(int32_t DeviceId) { in __tgt_rtl_init_device() argument
1588 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_init_device()
1591 return DeviceRTL.initDevice(DeviceId); in __tgt_rtl_init_device()
1594 int32_t __tgt_rtl_deinit_device(int32_t DeviceId) { in __tgt_rtl_deinit_device() argument
1595 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_deinit_device()
1598 return DeviceRTL.deinitDevice(DeviceId); in __tgt_rtl_deinit_device()
1601 __tgt_target_table *__tgt_rtl_load_binary(int32_t DeviceId, in __tgt_rtl_load_binary() argument
1603 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_load_binary()
1605 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_load_binary()
1608 return DeviceRTL.loadBinary(DeviceId, Image); in __tgt_rtl_load_binary()
1611 void *__tgt_rtl_data_alloc(int32_t DeviceId, int64_t Size, void *, in __tgt_rtl_data_alloc() argument
1613 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_alloc()
1615 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_data_alloc()
1618 return DeviceRTL.dataAlloc(DeviceId, Size, (TargetAllocTy)Kind); in __tgt_rtl_data_alloc()
1621 int32_t __tgt_rtl_data_submit(int32_t DeviceId, void *TgtPtr, void *HstPtr, in __tgt_rtl_data_submit() argument
1623 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_submit()
1628 __tgt_rtl_data_submit_async(DeviceId, TgtPtr, HstPtr, Size, &AsyncInfo); in __tgt_rtl_data_submit()
1632 return __tgt_rtl_synchronize(DeviceId, &AsyncInfo); in __tgt_rtl_data_submit()
1635 int32_t __tgt_rtl_data_submit_async(int32_t DeviceId, void *TgtPtr, in __tgt_rtl_data_submit_async() argument
1638 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_submit_async()
1641 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_data_submit_async()
1644 return DeviceRTL.dataSubmit(DeviceId, TgtPtr, HstPtr, Size, AsyncInfoPtr); in __tgt_rtl_data_submit_async()
1647 int32_t __tgt_rtl_data_retrieve(int32_t DeviceId, void *HstPtr, void *TgtPtr, in __tgt_rtl_data_retrieve() argument
1649 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_retrieve()
1654 __tgt_rtl_data_retrieve_async(DeviceId, HstPtr, TgtPtr, Size, &AsyncInfo); in __tgt_rtl_data_retrieve()
1658 return __tgt_rtl_synchronize(DeviceId, &AsyncInfo); in __tgt_rtl_data_retrieve()
1661 int32_t __tgt_rtl_data_retrieve_async(int32_t DeviceId, void *HstPtr, in __tgt_rtl_data_retrieve_async() argument
1664 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_retrieve_async()
1667 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_data_retrieve_async()
1670 return DeviceRTL.dataRetrieve(DeviceId, HstPtr, TgtPtr, Size, AsyncInfoPtr); in __tgt_rtl_data_retrieve_async()
1702 int32_t __tgt_rtl_data_delete(int32_t DeviceId, void *TgtPtr) { in __tgt_rtl_data_delete() argument
1703 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_data_delete()
1705 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_data_delete()
1708 return DeviceRTL.dataDelete(DeviceId, TgtPtr); in __tgt_rtl_data_delete()
1711 int32_t __tgt_rtl_run_target_team_region(int32_t DeviceId, void *TgtEntryPtr, in __tgt_rtl_run_target_team_region() argument
1716 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_run_target_team_region()
1721 DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, TeamNum, ThreadLimit, in __tgt_rtl_run_target_team_region()
1726 return __tgt_rtl_synchronize(DeviceId, &AsyncInfo); in __tgt_rtl_run_target_team_region()
1730 int32_t DeviceId, void *TgtEntryPtr, void **TgtArgs, ptrdiff_t *TgtOffsets, in __tgt_rtl_run_target_team_region_async() argument
1733 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_run_target_team_region_async()
1735 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_run_target_team_region_async()
1738 return DeviceRTL.runTargetTeamRegion(DeviceId, TgtEntryPtr, TgtArgs, in __tgt_rtl_run_target_team_region_async()
1743 int32_t __tgt_rtl_run_target_region(int32_t DeviceId, void *TgtEntryPtr, in __tgt_rtl_run_target_region() argument
1746 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_run_target_region()
1751 DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, &AsyncInfo); in __tgt_rtl_run_target_region()
1755 return __tgt_rtl_synchronize(DeviceId, &AsyncInfo); in __tgt_rtl_run_target_region()
1758 int32_t __tgt_rtl_run_target_region_async(int32_t DeviceId, void *TgtEntryPtr, in __tgt_rtl_run_target_region_async() argument
1762 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_run_target_region_async()
1765 DeviceId, TgtEntryPtr, TgtArgs, TgtOffsets, ArgNum, in __tgt_rtl_run_target_region_async()
1770 int32_t __tgt_rtl_synchronize(int32_t DeviceId, in __tgt_rtl_synchronize() argument
1772 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_synchronize()
1776 return DeviceRTL.synchronize(DeviceId, AsyncInfoPtr); in __tgt_rtl_synchronize()
1784 void __tgt_rtl_print_device_info(int32_t DeviceId) { in __tgt_rtl_print_device_info() argument
1785 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_print_device_info()
1787 DeviceRTL.printDeviceInfo(DeviceId); in __tgt_rtl_print_device_info()
1790 int32_t __tgt_rtl_create_event(int32_t DeviceId, void **Event) { in __tgt_rtl_create_event() argument
1793 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_create_event()
1796 return DeviceRTL.createEvent(DeviceId, Event); in __tgt_rtl_create_event()
1799 int32_t __tgt_rtl_record_event(int32_t DeviceId, void *EventPtr, in __tgt_rtl_record_event() argument
1808 int32_t __tgt_rtl_wait_event(int32_t DeviceId, void *EventPtr, in __tgt_rtl_wait_event() argument
1810 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_wait_event()
1814 if (!AsyncInfoPtr->Queue && DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_wait_event()
1816 return DeviceRTL.waitEvent(DeviceId, AsyncInfoPtr, EventPtr); in __tgt_rtl_wait_event()
1819 int32_t __tgt_rtl_sync_event(int32_t DeviceId, void *EventPtr) { in __tgt_rtl_sync_event() argument
1825 int32_t __tgt_rtl_destroy_event(int32_t DeviceId, void *EventPtr) { in __tgt_rtl_destroy_event() argument
1828 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_destroy_event()
1831 return DeviceRTL.destroyEvent(DeviceId, EventPtr); in __tgt_rtl_destroy_event()
1834 int32_t __tgt_rtl_release_async_info(int32_t DeviceId, in __tgt_rtl_release_async_info() argument
1836 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_release_async_info()
1839 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_release_async_info()
1842 return DeviceRTL.releaseAsyncInfo(DeviceId, AsyncInfo); in __tgt_rtl_release_async_info()
1845 int32_t __tgt_rtl_init_async_info(int32_t DeviceId, in __tgt_rtl_init_async_info() argument
1847 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_init_async_info()
1850 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_init_async_info()
1853 return DeviceRTL.initAsyncInfo(DeviceId, AsyncInfo); in __tgt_rtl_init_async_info()
1856 int32_t __tgt_rtl_init_device_info(int32_t DeviceId, in __tgt_rtl_init_device_info() argument
1859 assert(DeviceRTL.isValidDeviceId(DeviceId) && "device_id is invalid"); in __tgt_rtl_init_device_info()
1862 if (DeviceRTL.setContext(DeviceId) != OFFLOAD_SUCCESS) in __tgt_rtl_init_device_info()
1865 return DeviceRTL.initDeviceInfo(DeviceId, DeviceInfoPtr, ErrStr); in __tgt_rtl_init_device_info()