1 ///
2 /// Perform several driver tests for OpenMP offloading
3 ///
4 
5 // REQUIRES: x86-registered-target
6 // REQUIRES: powerpc-registered-target
7 // REQUIRES: nvptx-registered-target
8 // REQUIRES: amdgpu-registered-target
9 
10 // UNSUPPORTED: aix
11 
12 /// ###########################################################################
13 
14 /// Check -Xopenmp-target uses one of the archs provided when several archs are used.
15 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
16 // RUN:          -fno-openmp-new-driver -Xopenmp-target -march=sm_35 -Xopenmp-target -march=sm_60 %s 2>&1 \
17 // RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-ARCHS %s
18 
19 // CHK-FOPENMP-TARGET-ARCHS: ptxas{{.*}}" "--gpu-name" "sm_60"
20 // CHK-FOPENMP-TARGET-ARCHS: nvlink{{.*}}" "-arch" "sm_60"
21 
22 /// ###########################################################################
23 
24 /// Check -Xopenmp-target -march=sm_35 works as expected when two triples are present.
25 // RUN:   %clang -### -fopenmp=libomp -fno-openmp-new-driver \
26 // RUN:          -fopenmp-targets=powerpc64le-ibm-linux-gnu,nvptx64-nvidia-cuda \
27 // RUN:          -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_35 %s 2>&1 \
28 // RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-COMPILATION %s
29 
30 // CHK-FOPENMP-TARGET-COMPILATION: ptxas{{.*}}" "--gpu-name" "sm_35"
31 // CHK-FOPENMP-TARGET-COMPILATION: nvlink{{.*}}" "-arch" "sm_35"
32 
33 /// ###########################################################################
34 
35 /// Check cubin file generation and usage by nvlink
36 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp \
37 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda -save-temps %s 2>&1 \
38 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-NVLINK %s
39 /// Check cubin file generation and usage by nvlink when toolchain has BindArchAction
40 // RUN:   %clang -### --target=x86_64-apple-darwin17.0.0 -fopenmp=libomp \
41 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
42 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-NVLINK %s
43 
44 // CHK-CUBIN-NVLINK: clang{{.*}}" {{.*}}"-fopenmp-is-device" {{.*}}"-o" "[[PTX:.*\.s]]"
45 // CHK-CUBIN-NVLINK-NEXT: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX]]"
46 // CHK-CUBIN-NVLINK-NEXT: nvlink{{.*}}" {{.*}}"[[CUBIN]]"
47 
48 /// ###########################################################################
49 
50 /// Check unbundlink of assembly file, cubin file generation and usage by nvlink
51 // RUN:   touch %t.s
52 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
53 // RUN:          -fno-openmp-new-driver -save-temps %t.s 2>&1 \
54 // RUN:   | FileCheck -check-prefix=CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK %s
55 
56 /// Use DAG to ensure that assembly file has been unbundled.
57 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX:.*\.s]]"
58 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG: clang-offload-bundler{{.*}}" "-type=s" {{.*}}"-output={{.*}}[[PTX]]
59 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG-SAME: "-unbundle"
60 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK: nvlink{{.*}}" {{.*}}"[[CUBIN]]"
61 
62 /// ###########################################################################
63 
64 /// Check cubin file generation and bundling
65 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
66 // RUN:          -fno-openmp-new-driver -save-temps %s -c 2>&1 \
67 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-CUBIN-BUNDLING %s
68 
69 // CHK-PTXAS-CUBIN-BUNDLING: clang{{.*}}" "-o" "[[PTX:.*\.s]]"
70 // CHK-PTXAS-CUBIN-BUNDLING-NEXT: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX]]"
71 // CHK-PTXAS-CUBIN-BUNDLING: clang-offload-bundler{{.*}}" "-type=o" {{.*}}"-input={{.*}}[[CUBIN]]
72 
73 /// ###########################################################################
74 
75 /// Check cubin file unbundling and usage by nvlink
76 // RUN:   touch %t.o
77 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
78 // RUN:          -fno-openmp-new-driver -save-temps %t.o %S/Inputs/in.so 2>&1 \
79 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-UNBUNDLING-NVLINK %s
80 
81 /// Use DAG to ensure that cubin file has been unbundled.
82 // CHK-CUBIN-UNBUNDLING-NVLINK-NOT: clang-offload-bundler{{.*}}" "-type=o"{{.*}}in.so
83 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG: nvlink{{.*}}" {{.*}}"[[CUBIN:.*\.cubin]]"
84 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG: clang-offload-bundler{{.*}}" "-type=o" {{.*}}"-output={{.*}}[[CUBIN]]
85 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG-SAME: "-unbundle"
86 // CHK-CUBIN-UNBUNDLING-NVLINK-NOT: clang-offload-bundler{{.*}}" "-type=o"{{.*}}in.so
87 
88 /// ###########################################################################
89 
90 /// Check cubin file generation and usage by nvlink
91 // RUN:   touch %t1.o
92 // RUN:   touch %t2.o
93 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp \
94 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %t1.o %t2.o 2>&1 \
95 // RUN:   | FileCheck -check-prefix=CHK-TWOCUBIN %s
96 /// Check cubin file generation and usage by nvlink when toolchain has BindArchAction
97 // RUN:   %clang -### --target=x86_64-apple-darwin17.0.0 -fopenmp=libomp \
98 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %t1.o %t2.o 2>&1 \
99 // RUN:   | FileCheck -check-prefix=CHK-TWOCUBIN %s
100 
101 // CHK-TWOCUBIN: nvlink{{.*}}openmp-offload-{{.*}}.cubin" "{{.*}}openmp-offload-{{.*}}.cubin"
102 
103 /// ###########################################################################
104 
105 /// Check PTXAS is passed -c flag when offloading to an NVIDIA device using OpenMP.
106 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
107 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-DEFAULT %s
108 
109 // CHK-PTXAS-DEFAULT: ptxas{{.*}}" "-c"
110 
111 /// ###########################################################################
112 
113 /// PTXAS is passed -c flag by default when offloading to an NVIDIA device using OpenMP - disable it.
114 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -fnoopenmp-relocatable-target \
115 // RUN:          -save-temps %s 2>&1 \
116 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-NORELO %s
117 
118 // CHK-PTXAS-NORELO-NOT: ptxas{{.*}}" "-c"
119 
120 /// ###########################################################################
121 
122 /// PTXAS is passed -c flag by default when offloading to an NVIDIA device using OpenMP
123 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
124 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-relocatable-target \
125 // RUN:          -save-temps %s 2>&1 \
126 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-RELO %s
127 
128 // CHK-PTXAS-RELO: ptxas{{.*}}" "-c"
129 
130 /// ###########################################################################
131 
132 /// Check that error is not thrown by toolchain when no cuda lib flag is used.
133 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
134 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 \
135 // RUN:   -nocudalib -fopenmp-relocatable-target -save-temps %s 2>&1 \
136 // RUN:   | FileCheck -check-prefix=CHK-FLAG-NOLIBDEVICE %s
137 
138 // CHK-FLAG-NOLIBDEVICE-NOT: error:{{.*}}sm_60
139 
140 /// ###########################################################################
141 
142 /// Check that error is not thrown by toolchain when no cuda lib device is found when using -S.
143 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
144 // RUN:   %clang -### -S -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 \
145 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
146 // RUN:   | FileCheck -check-prefix=CHK-NOLIBDEVICE %s
147 
148 // CHK-NOLIBDEVICE-NOT: error:{{.*}}sm_60
149 
150 /// ###########################################################################
151 
152 /// Check that the runtime bitcode library is part of the compile line.
153 /// Create a bogus bitcode library and specify it with libomptarget-nvptx-bc-path
154 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
155 // RUN:   --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-nvptx-test.bc \
156 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
157 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
158 // RUN:   | FileCheck -check-prefix=CHK-BCLIB %s
159 
160 /// Specify the directory containing the bitcode lib, check clang picks the right one
161 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
162 // RUN:   --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget \
163 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
164 // RUN:   -fopenmp-relocatable-target -save-temps \
165 // RUN:   %s 2>&1 | FileCheck -check-prefix=CHK-BCLIB-DIR %s
166 
167 /// Create a bogus bitcode library and find it with LIBRARY_PATH
168 // RUN:   env LIBRARY_PATH=%S/Inputs/libomptarget/subdir %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
169 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
170 // RUN:   -fopenmp-relocatable-target -save-temps \
171 // RUN:   %s 2>&1 | FileCheck -check-prefix=CHK-ENV-BCLIB %s
172 
173 // CHK-BCLIB: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}libomptarget-nvptx-test.bc
174 // CHK-BCLIB-DIR: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}libomptarget{{/|\\\\}}libomptarget-nvptx-sm_35.bc
175 // CHK-ENV-BCLIB: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}subdir{{/|\\\\}}libomptarget-nvptx-sm_35.bc
176 // CHK-BCLIB-NOT: {{error:|warning:}}
177 
178 /// ###########################################################################
179 
180 /// Check that the warning is thrown when the libomptarget bitcode library is not found.
181 /// Libomptarget requires sm_35 or newer so an sm_35 bitcode library should never exist.
182 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
183 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
184 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
185 // RUN:   | FileCheck -check-prefix=CHK-BCLIB-WARN %s
186 
187 // CHK-BCLIB-WARN: no library 'libomptarget-nvptx-sm_35.bc' found in the default clang lib directory or in LIBRARY_PATH; use '--libomptarget-nvptx-bc-path' to specify nvptx bitcode library
188 
189 /// ###########################################################################
190 
191 /// Check that the error is thrown when the libomptarget bitcode library does not exist.
192 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
193 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
194 // RUN:   --libomptarget-nvptx-bc-path=not-exist.bc \
195 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
196 // RUN:   | FileCheck -check-prefix=CHK-BCLIB-ERROR %s
197 
198 // CHK-BCLIB-ERROR: bitcode library 'not-exist.bc' does not exist
199 
200 /// ###########################################################################
201 
202 /// Check that the error is thrown when CUDA 9.1 or lower version is used.
203 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
204 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_90/usr/local/cuda \
205 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
206 // RUN:   | FileCheck -check-prefix=CHK-CUDA-VERSION-ERROR %s
207 
208 // CHK-CUDA-VERSION-ERROR: NVPTX target requires CUDA 9.2 or above; CUDA 9.0 detected
209 
210 /// Check that debug info is emitted in dwarf-2
211 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O1 --no-cuda-noopt-device-debug 2>&1 \
212 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
213 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 2>&1 \
214 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
215 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 --no-cuda-noopt-device-debug 2>&1 \
216 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
217 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g0 2>&1 \
218 // RUN:   | FileCheck -check-prefix=NO_DEBUG %s
219 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb0 -O3 --cuda-noopt-device-debug 2>&1 \
220 // RUN:   | FileCheck -check-prefix=NO_DEBUG %s
221 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -gline-directives-only 2>&1 \
222 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
223 
224 // DEBUG_DIRECTIVES-NOT: warning: debug
225 // NO_DEBUG-NOT: warning: debug
226 // NO_DEBUG: "-fopenmp-is-device"
227 // NO_DEBUG-NOT: "-debug-info-kind=
228 // NO_DEBUG: ptxas
229 // DEBUG_DIRECTIVES: "-triple" "nvptx64-nvidia-cuda"
230 // DEBUG_DIRECTIVES-SAME: "-debug-info-kind=line-directives-only"
231 // DEBUG_DIRECTIVES-SAME: "-fopenmp-is-device"
232 // DEBUG_DIRECTIVES: ptxas
233 // DEBUG_DIRECTIVES: "-lineinfo"
234 // NO_DEBUG-NOT: "-g"
235 // NO_DEBUG: nvlink
236 // NO_DEBUG-NOT: "-g"
237 
238 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O0 --no-cuda-noopt-device-debug 2>&1 \
239 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
240 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g 2>&1 \
241 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
242 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O0 --cuda-noopt-device-debug 2>&1 \
243 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
244 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 --cuda-noopt-device-debug 2>&1 \
245 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
246 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g2 2>&1 \
247 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
248 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb2 -O0 --cuda-noopt-device-debug 2>&1 \
249 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
250 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g3 -O3 --cuda-noopt-device-debug 2>&1 \
251 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
252 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb3 -O2 --cuda-noopt-device-debug 2>&1 \
253 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
254 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -gline-tables-only 2>&1 \
255 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
256 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb1 -O2 --cuda-noopt-device-debug 2>&1 \
257 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
258 
259 // HAS_DEBUG-NOT: warning: debug
260 // HAS_DEBUG: "-triple" "nvptx64-nvidia-cuda"
261 // HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
262 // HAS_DEBUG-SAME: "-dwarf-version=2"
263 // HAS_DEBUG-SAME: "-fopenmp-is-device"
264 // HAS_DEBUG: ptxas
265 // HAS_DEBUG-SAME: "-g"
266 // HAS_DEBUG-SAME: "--dont-merge-basicblocks"
267 // HAS_DEBUG-SAME: "--return-at-end"
268 // HAS_DEBUG: nvlink
269 // HAS_DEBUG-SAME: "-g"
270 
271 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-mode 2>&1 \
272 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
273 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-mode -fopenmp-cuda-mode 2>&1 \
274 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
275 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-mode 2>&1 \
276 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
277 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-mode -fopenmp-cuda-mode 2>&1 \
278 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
279 // CUDA_MODE: "-cc1"{{.*}}"-triple" "{{nvptx64-nvidia-cuda|amdgcn-amd-amdhsa}}"
280 // CUDA_MODE-SAME: "-fopenmp-cuda-mode"
281 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-mode 2>&1 \
282 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
283 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-mode -fno-openmp-cuda-mode 2>&1 \
284 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
285 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-mode 2>&1 \
286 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
287 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-mode -fno-openmp-cuda-mode 2>&1 \
288 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
289 // NO_CUDA_MODE-NOT: "-{{fno-|f}}openmp-cuda-mode"
290 
291 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-force-full-runtime 2>&1 \
292 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
293 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-force-full-runtime -fopenmp-cuda-force-full-runtime 2>&1 \
294 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
295 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-force-full-runtime 2>&1 \
296 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
297 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-force-full-runtime -fopenmp-cuda-force-full-runtime 2>&1 \
298 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
299 // FULL_RUNTIME: "-cc1"{{.*}}"-triple" "{{nvptx64-nvidia-cuda|amdgcn-amd-amdhsa}}"
300 // FULL_RUNTIME-SAME: "-fopenmp-cuda-force-full-runtime"
301 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-force-full-runtime 2>&1 \
302 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
303 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-force-full-runtime -fno-openmp-cuda-force-full-runtime 2>&1 \
304 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
305 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-force-full-runtime 2>&1 \
306 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
307 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-force-full-runtime -fno-openmp-cuda-force-full-runtime 2>&1 \
308 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
309 // NO_FULL_RUNTIME-NOT: "-{{fno-|f}}openmp-cuda-force-full-runtime"
310 
311 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-teams-reduction-recs-num=2048 2>&1 \
312 // RUN:   | FileCheck -check-prefix=CUDA_RED_RECS %s
313 // CUDA_RED_RECS: "-cc1"{{.*}}"-triple" "nvptx64-nvidia-cuda"
314 // CUDA_RED_RECS-SAME: "-fopenmp-cuda-teams-reduction-recs-num=2048"
315 
316 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
317 // RUN:   | FileCheck -check-prefix=OPENMP_NVPTX_WRAPPERS %s
318 // OPENMP_NVPTX_WRAPPERS: "-cc1"{{.*}}"-triple" "nvptx64-nvidia-cuda"
319 // OPENMP_NVPTX_WRAPPERS-SAME: "-internal-isystem" "{{.*}}openmp_wrappers"
320 
321 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
322 // RUN:          -save-temps -ccc-print-bindings %s -o openmp-offload-gpu 2>&1 \
323 // RUN:   | FileCheck -check-prefix=SAVE_TEMPS_NAMES %s
324 
325 // SAVE_TEMPS_NAMES-NOT: "GNU::Linker"{{.*}}["[[SAVE_TEMPS_INPUT1:.*\.o]]", "[[SAVE_TEMPS_INPUT1]]"]
326 
327 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64 -Xopenmp-target=nvptx64 -march=sm_35 \
328 // RUN:          -save-temps %s -o openmp-offload-gpu 2>&1 \
329 // RUN:   | FileCheck -check-prefix=TRIPLE %s
330 
331 // TRIPLE: "-triple" "nvptx64-nvidia-cuda"
332 // TRIPLE: "-target-cpu" "sm_35"
333