1##===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7##===----------------------------------------------------------------------===## 8# 9# Build the DeviceRTL for all toolchains that are available 10# 11##===----------------------------------------------------------------------===## 12 13set(LIBOMPTARGET_BUILD_DEVICERTL_BCLIB TRUE CACHE BOOL 14 "Can be set to false to disable building this library.") 15 16if (NOT LIBOMPTARGET_BUILD_DEVICERTL_BCLIB) 17 libomptarget_say("Not building DeviceRTL: Disabled by LIBOMPTARGET_BUILD_DEVICERTL_BCLIB") 18 return() 19endif() 20 21# Check to ensure the host system is a supported host architecture. 22if(NOT ${CMAKE_SIZEOF_VOID_P} EQUAL "8") 23 libomptarget_say("Not building DeviceRTL: Runtime does not support 32-bit hosts") 24 return() 25endif() 26 27if (LLVM_DIR) 28 # Builds that use pre-installed LLVM have LLVM_DIR set. 29 # A standalone or LLVM_ENABLE_RUNTIMES=openmp build takes this route 30 find_program(CLANG_TOOL clang PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 31 find_program(LINK_TOOL llvm-link PATHS ${LLVM_TOOLS_BINARY_DIR} 32 NO_DEFAULT_PATH) 33 find_program(OPT_TOOL opt PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH) 34 if ((NOT CLANG_TOOL) OR (NOT LINK_TOOL) OR (NOT OPT_TOOL)) 35 libomptarget_say("Not building DeviceRTL. Missing clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} or opt: ${OPT_TOOL}") 36 return() 37 else() 38 libomptarget_say("Building DeviceRTL. Using clang: ${CLANG_TOOL}, llvm-link: ${LINK_TOOL} and opt: ${OPT_TOOL}") 39 endif() 40elseif (LLVM_TOOL_CLANG_BUILD AND NOT CMAKE_CROSSCOMPILING AND NOT OPENMP_STANDALONE_BUILD) 41 # LLVM in-tree builds may use CMake target names to discover the tools. 42 # A LLVM_ENABLE_PROJECTS=openmp build takes this route 43 set(CLANG_TOOL $<TARGET_FILE:clang>) 44 set(LINK_TOOL $<TARGET_FILE:llvm-link>) 45 set(OPT_TOOL $<TARGET_FILE:opt>) 46 libomptarget_say("Building DeviceRTL. Using clang from in-tree build") 47else() 48 libomptarget_say("Not building DeviceRTL. No appropriate clang found") 49 return() 50endif() 51 52set(devicertl_base_directory ${CMAKE_CURRENT_SOURCE_DIR}) 53set(include_directory ${devicertl_base_directory}/include) 54set(source_directory ${devicertl_base_directory}/src) 55 56set(all_capabilities 35 37 50 52 53 60 61 62 70 72 75 80 86) 57 58set(LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES ${all_capabilities} CACHE STRING 59 "List of CUDA Compute Capabilities to be used to compile the NVPTX DeviceRTL.") 60string(TOLOWER ${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES} LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES) 61 62if (LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES STREQUAL "all") 63 set(nvptx_sm_list ${all_capabilities}) 64elseif(LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES STREQUAL "auto") 65 if (NOT LIBOMPTARGET_DEP_CUDA_FOUND) 66 libomptarget_error_say("[NVPTX] Cannot auto detect compute capability as CUDA not found.") 67 endif() 68 set(nvptx_sm_list ${LIBOMPTARGET_DEP_CUDA_ARCH}) 69else() 70 string(REPLACE "," ";" nvptx_sm_list "${LIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES}") 71endif() 72 73# Check all SM values 74foreach(sm ${nvptx_sm_list}) 75 if (NOT ${sm} IN_LIST all_capabilities) 76 libomptarget_warning_say("[NVPTX] Compute capability ${sm} is not supported. Make sure clang can work with it.") 77 endif() 78endforeach() 79 80set(amdgpu_mcpus gfx700 gfx701 gfx801 gfx803 gfx900 gfx902 gfx906 gfx908 gfx90a gfx90c gfx940 gfx1010 gfx1030 gfx1031 gfx1032 gfx1033 gfx1034 gfx1035 gfx1036) 81if (DEFINED LIBOMPTARGET_AMDGCN_GFXLIST) 82 set(amdgpu_mcpus ${LIBOMPTARGET_AMDGCN_GFXLIST}) 83endif() 84 85# Activate RTL message dumps if requested by the user. 86set(LIBOMPTARGET_DEVICE_DEBUG FALSE CACHE BOOL 87 "Activate DeviceRTL debug messages.") 88 89set(include_files 90 ${include_directory}/Configuration.h 91 ${include_directory}/Debug.h 92 ${include_directory}/Interface.h 93 ${include_directory}/Mapping.h 94 ${include_directory}/State.h 95 ${include_directory}/Synchronization.h 96 ${include_directory}/Types.h 97 ${include_directory}/Utils.h 98) 99 100set(src_files 101 ${source_directory}/Configuration.cpp 102 ${source_directory}/Debug.cpp 103 ${source_directory}/Kernel.cpp 104 ${source_directory}/Mapping.cpp 105 ${source_directory}/Misc.cpp 106 ${source_directory}/Parallelism.cpp 107 ${source_directory}/Reduction.cpp 108 ${source_directory}/State.cpp 109 ${source_directory}/Synchronization.cpp 110 ${source_directory}/Tasking.cpp 111 ${source_directory}/Utils.cpp 112 ${source_directory}/Workshare.cpp 113) 114 115set(clang_opt_flags -O3 -mllvm -openmp-opt-disable -DSHARED_SCRATCHPAD_SIZE=512) 116set(link_opt_flags -O3 -openmp-opt-disable) 117set(link_export_flag -passes=internalize -internalize-public-api-file=${source_directory}/exports) 118 119# Prepend -I to each list element 120set (LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL "${LIBOMPTARGET_LLVM_INCLUDE_DIRS}") 121list(TRANSFORM LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL PREPEND "-I") 122 123# Set flags for LLVM Bitcode compilation. 124set(bc_flags -c -emit-llvm -std=c++17 -fvisibility=hidden 125 ${clang_opt_flags} --offload-device-only 126 -nocudalib -nogpulib -nostdinc 127 -fopenmp -fopenmp-cuda-mode 128 -I${include_directory} 129 -I${devicertl_base_directory}/../include 130 ${LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL} 131) 132 133if(${LIBOMPTARGET_DEVICE_DEBUG}) 134 list(APPEND bc_flags -DOMPTARGET_DEBUG=-1) 135else() 136 list(APPEND bc_flags -DOMPTARGET_DEBUG=0) 137endif() 138 139function(compileDeviceRTLLibrary target_cpu target_name) 140 set(target_bc_flags ${ARGN}) 141 142 set(bc_files "") 143 foreach(src ${src_files}) 144 get_filename_component(infile ${src} ABSOLUTE) 145 get_filename_component(outfile ${src} NAME) 146 set(outfile "${outfile}-${target_cpu}.bc") 147 148 add_custom_command(OUTPUT ${outfile} 149 COMMAND ${CLANG_TOOL} 150 ${bc_flags} 151 --offload-arch=${target_cpu} 152 ${target_bc_flags} 153 ${infile} -o ${outfile} 154 DEPENDS ${infile} ${include_files} 155 IMPLICIT_DEPENDS CXX ${infile} 156 COMMENT "Building LLVM bitcode ${outfile}" 157 VERBATIM 158 ) 159 if("${CLANG_TOOL}" STREQUAL "$<TARGET_FILE:clang>") 160 # Add a file-level dependency to ensure that clang is up-to-date. 161 # By default, add_custom_command only builds clang if the 162 # executable is missing. 163 add_custom_command(OUTPUT ${outfile} 164 DEPENDS clang 165 APPEND 166 ) 167 endif() 168 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${outfile}) 169 170 list(APPEND bc_files ${outfile}) 171 endforeach() 172 173 set(bclib_name "libomptarget-${target_name}-${target_cpu}.bc") 174 175 # Link to a bitcode library. 176 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 177 COMMAND ${LINK_TOOL} 178 -o ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} ${bc_files} 179 DEPENDS ${bc_files} 180 COMMENT "Linking LLVM bitcode ${bclib_name}" 181 ) 182 183 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 184 COMMAND ${OPT_TOOL} ${link_export_flag} ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 185 -o ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 186 DEPENDS ${source_directory}/exports ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 187 COMMENT "Internalizing LLVM bitcode ${bclib_name}" 188 ) 189 190 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 191 COMMAND ${OPT_TOOL} ${link_opt_flags} ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 192 -o ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 193 DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/internalized_${bclib_name} 194 COMMENT "Optimizing LLVM bitcode ${bclib_name}" 195 ) 196 197 # Add a file-level dependency to ensure that llvm-link and opt are up-to-date. 198 # By default, add_custom_command only builds the tool if the executable is missing 199 if("${LINK_TOOL}" STREQUAL "$<TARGET_FILE:llvm-link>") 200 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/linked_${bclib_name} 201 DEPENDS llvm-link 202 APPEND) 203 endif() 204 if("${OPT_TOOL}" STREQUAL "$<TARGET_FILE:opt>") 205 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 206 DEPENDS opt 207 APPEND) 208 endif() 209 210 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${bclib_name}) 211 212 set(bclib_target_name "omptarget-${target_name}-${target_cpu}-bc") 213 214 add_custom_target(${bclib_target_name} ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name}) 215 216 # Copy library to destination. 217 add_custom_command(TARGET ${bclib_target_name} POST_BUILD 218 COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} 219 ${LIBOMPTARGET_LIBRARY_DIR}) 220 add_dependencies("omptarget.devicertl.${target_name}" "${bclib_target_name}") 221 222 # Install bitcode library under the lib destination folder. 223 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${bclib_name} DESTINATION "${OPENMP_INSTALL_LIBDIR}") 224endfunction() 225 226# Generate a Bitcode library for all the compute capabilities the user requested 227add_custom_target(omptarget.devicertl.nvptx) 228foreach(sm ${nvptx_sm_list}) 229 compileDeviceRTLLibrary(sm_${sm} nvptx -fopenmp-targets=nvptx64-nvidia-cuda -DLIBOMPTARGET_BC_TARGET --cuda-feature=+ptx61) 230endforeach() 231 232add_custom_target(omptarget.devicertl.amdgpu) 233foreach(mcpu ${amdgpu_mcpus}) 234 compileDeviceRTLLibrary(${mcpu} amdgpu -fopenmp-targets=amdgcn-amd-amdhsa -DLIBOMPTARGET_BC_TARGET -D__AMDGCN__ -nogpulib) 235endforeach() 236 237# Set the flags to build the device runtime from clang. 238set(clang_lib_flags -fopenmp -fopenmp-cuda-mode -foffload-lto -fvisibility=hidden -Xopenmp-target=nvptx64-nvidia-cuda --cuda-feature=+ptx61 -nocudalib -nogpulib -nostdinc ${clang_opt_flags}) 239foreach(arch ${nvptx_sm_list}) 240 set(clang_lib_flags ${clang_lib_flags} --offload-arch=sm_${arch}) 241endforeach() 242foreach(arch ${amdgpu_mcpus}) 243 set(clang_lib_flags ${clang_lib_flags} --offload-arch=${arch}) 244endforeach() 245 246# Build the static library version of the device runtime. 247# first create an object target 248add_library(omptarget.devicertl.all_objs OBJECT IMPORTED) 249foreach(src ${src_files}) 250 get_filename_component(infile ${src} ABSOLUTE) 251 get_filename_component(outfile ${src} NAME) 252 set(outfile "${outfile}.o") 253 set(outfile_full_path "${CMAKE_CURRENT_BINARY_DIR}/${outfile}") 254 255 add_custom_command(OUTPUT ${outfile_full_path} 256 COMMAND ${CLANG_TOOL} ${clang_lib_flags} --std=c++17 -c 257 -o ${outfile_full_path} 258 -I${include_directory} 259 -I${devicertl_base_directory}/../include 260 ${LIBOMPTARGET_LLVM_INCLUDE_DIRS_DEVICERTL} 261 ${infile} 262 DEPENDS ${infile} ${include_files} 263 IMPLICIT_DEPENDS CXX ${infile} 264 COMMENT "Building device code ${outfile}" 265 VERBATIM 266 ) 267 if("${CLANG_TOOL}" STREQUAL "$<TARGET_FILE:clang>") 268 # Add a file-level dependency to ensure that clang is up-to-date. 269 # By default, add_custom_command only builds clang if the 270 # executable is missing. 271 add_custom_command(OUTPUT ${outfile_full_path} 272 DEPENDS clang 273 APPEND 274 ) 275 endif() 276 set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${outfile_full_path}) 277 278 set_property(TARGET omptarget.devicertl.all_objs APPEND PROPERTY IMPORTED_OBJECTS ${outfile_full_path}) 279endforeach() 280 281# second archive all the object files into a static library 282add_library(omptarget.devicertl STATIC) 283set_target_properties(omptarget.devicertl PROPERTIES LINKER_LANGUAGE CXX) 284target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs) 285 286install(TARGETS omptarget.devicertl ARCHIVE DESTINATION ${OPENMP_INSTALL_LIBDIR}) 287