clang/test/Driver/openmp-offload-gpu.c

   1 ///
   2 /// Perform several driver tests for OpenMP offloading
   3 ///
   4
   5 // REQUIRES: x86-registered-target
   6 // REQUIRES: powerpc-registered-target
   7 // REQUIRES: nvptx-registered-target
   8 // REQUIRES: amdgpu-registered-target
   9
  10 // UNSUPPORTED: aix
  11
  12 /// ###########################################################################
  13
  14 /// Check -Xopenmp-target uses one of the archs provided when several archs are used.
  15 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
  16 // RUN:          -fno-openmp-new-driver -Xopenmp-target -march=sm_35 -Xopenmp-target -march=sm_60 %s 2>&1 \
  17 // RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-ARCHS %s
  18
  19 // CHK-FOPENMP-TARGET-ARCHS: ptxas{{.*}}" "--gpu-name" "sm_60"
  20 // CHK-FOPENMP-TARGET-ARCHS: nvlink{{.*}}" "-arch" "sm_60"
  21
  22 /// ###########################################################################
  23
  24 /// Check -Xopenmp-target -march=sm_35 works as expected when two triples are present.
  25 // RUN:   %clang -### -fopenmp=libomp -fno-openmp-new-driver \
  26 // RUN:          -fopenmp-targets=powerpc64le-ibm-linux-gnu,nvptx64-nvidia-cuda \
  27 // RUN:          -Xopenmp-target=nvptx64-nvidia-cuda -march=sm_35 %s 2>&1 \
  28 // RUN:   | FileCheck -check-prefix=CHK-FOPENMP-TARGET-COMPILATION %s
  29
  30 // CHK-FOPENMP-TARGET-COMPILATION: ptxas{{.*}}" "--gpu-name" "sm_35"
  31 // CHK-FOPENMP-TARGET-COMPILATION: nvlink{{.*}}" "-arch" "sm_35"
  32
  33 /// ###########################################################################
  34
  35 /// Check cubin file generation and usage by nvlink
  36 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp \
  37 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda -save-temps %s 2>&1 \
  38 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-NVLINK %s
  39 /// Check cubin file generation and usage by nvlink when toolchain has BindArchAction
  40 // RUN:   %clang -### --target=x86_64-apple-darwin17.0.0 -fopenmp=libomp \
  41 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
  42 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-NVLINK %s
  43
  44 // CHK-CUBIN-NVLINK: clang{{.*}}" {{.*}}"-fopenmp-is-device" {{.*}}"-o" "[[PTX:.*\.s]]"
  45 // CHK-CUBIN-NVLINK-NEXT: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX]]"
  46 // CHK-CUBIN-NVLINK-NEXT: nvlink{{.*}}" {{.*}}"[[CUBIN]]"
  47
  48 /// ###########################################################################
  49
  50 /// Check unbundlink of assembly file, cubin file generation and usage by nvlink
  51 // RUN:   touch %t.s
  52 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
  53 // RUN:          -fno-openmp-new-driver -save-temps %t.s 2>&1 \
  54 // RUN:   | FileCheck -check-prefix=CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK %s
  55
  56 /// Use DAG to ensure that assembly file has been unbundled.
  57 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX:.*\.s]]"
  58 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG: clang-offload-bundler{{.*}}" "-type=s" {{.*}}"-output={{.*}}[[PTX]]
  59 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK-DAG-SAME: "-unbundle"
  60 // CHK-UNBUNDLING-PTXAS-CUBIN-NVLINK: nvlink{{.*}}" {{.*}}"[[CUBIN]]"
  61
  62 /// ###########################################################################
  63
  64 /// Check cubin file generation and bundling
  65 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
  66 // RUN:          -fno-openmp-new-driver -save-temps %s -c 2>&1 \
  67 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-CUBIN-BUNDLING %s
  68
  69 // CHK-PTXAS-CUBIN-BUNDLING: clang{{.*}}" "-o" "[[PTX:.*\.s]]"
  70 // CHK-PTXAS-CUBIN-BUNDLING-NEXT: ptxas{{.*}}" "--output-file" "[[CUBIN:.*\.cubin]]" {{.*}}"[[PTX]]"
  71 // CHK-PTXAS-CUBIN-BUNDLING: clang-offload-bundler{{.*}}" "-type=o" {{.*}}"-input={{.*}}[[CUBIN]]
  72
  73 /// ###########################################################################
  74
  75 /// Check cubin file unbundling and usage by nvlink
  76 // RUN:   touch %t.o
  77 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
  78 // RUN:          -fno-openmp-new-driver -save-temps %t.o %S/Inputs/in.so 2>&1 \
  79 // RUN:   | FileCheck -check-prefix=CHK-CUBIN-UNBUNDLING-NVLINK %s
  80
  81 /// Use DAG to ensure that cubin file has been unbundled.
  82 // CHK-CUBIN-UNBUNDLING-NVLINK-NOT: clang-offload-bundler{{.*}}" "-type=o"{{.*}}in.so
  83 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG: nvlink{{.*}}" {{.*}}"[[CUBIN:.*\.cubin]]"
  84 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG: clang-offload-bundler{{.*}}" "-type=o" {{.*}}"-output={{.*}}[[CUBIN]]
  85 // CHK-CUBIN-UNBUNDLING-NVLINK-DAG-SAME: "-unbundle"
  86 // CHK-CUBIN-UNBUNDLING-NVLINK-NOT: clang-offload-bundler{{.*}}" "-type=o"{{.*}}in.so
  87
  88 /// ###########################################################################
  89
  90 /// Check cubin file generation and usage by nvlink
  91 // RUN:   touch %t1.o
  92 // RUN:   touch %t2.o
  93 // RUN:   %clang -### --target=powerpc64le-unknown-linux-gnu -fopenmp=libomp \
  94 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %t1.o %t2.o 2>&1 \
  95 // RUN:   | FileCheck -check-prefix=CHK-TWOCUBIN %s
  96 /// Check cubin file generation and usage by nvlink when toolchain has BindArchAction
  97 // RUN:   %clang -### --target=x86_64-apple-darwin17.0.0 -fopenmp=libomp \
  98 // RUN:          -fno-openmp-new-driver -fopenmp-targets=nvptx64-nvidia-cuda %t1.o %t2.o 2>&1 \
  99 // RUN:   | FileCheck -check-prefix=CHK-TWOCUBIN %s
 100
 101 // CHK-TWOCUBIN: nvlink{{.*}}openmp-offload-{{.*}}.cubin" "{{.*}}openmp-offload-{{.*}}.cubin"
 102
 103 /// ###########################################################################
 104
 105 /// Check PTXAS is passed -c flag when offloading to an NVIDIA device using OpenMP.
 106 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
 107 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-DEFAULT %s
 108
 109 // CHK-PTXAS-DEFAULT: ptxas{{.*}}" "-c"
 110
 111 /// ###########################################################################
 112
 113 /// PTXAS is passed -c flag by default when offloading to an NVIDIA device using OpenMP - disable it.
 114 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -fnoopenmp-relocatable-target \
 115 // RUN:          -save-temps %s 2>&1 \
 116 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-NORELO %s
 117
 118 // CHK-PTXAS-NORELO-NOT: ptxas{{.*}}" "-c"
 119
 120 /// ###########################################################################
 121
 122 /// PTXAS is passed -c flag by default when offloading to an NVIDIA device using OpenMP
 123 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
 124 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp-relocatable-target \
 125 // RUN:          -save-temps %s 2>&1 \
 126 // RUN:   | FileCheck -check-prefix=CHK-PTXAS-RELO %s
 127
 128 // CHK-PTXAS-RELO: ptxas{{.*}}" "-c"
 129
 130 /// ###########################################################################
 131
 132 /// Check that error is not thrown by toolchain when no cuda lib flag is used.
 133 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
 134 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 \
 135 // RUN:   -nocudalib -fopenmp-relocatable-target -save-temps %s 2>&1 \
 136 // RUN:   | FileCheck -check-prefix=CHK-FLAG-NOLIBDEVICE %s
 137
 138 // CHK-FLAG-NOLIBDEVICE-NOT: error:{{.*}}sm_60
 139
 140 /// ###########################################################################
 141
 142 /// Check that error is not thrown by toolchain when no cuda lib device is found when using -S.
 143 /// Check that the flag is passed when -fopenmp-relocatable-target is used.
 144 // RUN:   %clang -### -S -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 \
 145 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
 146 // RUN:   | FileCheck -check-prefix=CHK-NOLIBDEVICE %s
 147
 148 // CHK-NOLIBDEVICE-NOT: error:{{.*}}sm_60
 149
 150 /// ###########################################################################
 151
 152 /// Check that the runtime bitcode library is part of the compile line.
 153 /// Create a bogus bitcode library and specify it with libomptarget-nvptx-bc-path
 154 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 155 // RUN:   --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-nvptx-test.bc \
 156 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 157 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
 158 // RUN:   | FileCheck -check-prefix=CHK-BCLIB %s
 159
 160 /// Specify the directory containing the bitcode lib, check clang picks the right one
 161 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 162 // RUN:   --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget \
 163 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 164 // RUN:   -fopenmp-relocatable-target -save-temps \
 165 // RUN:   %s 2>&1 | FileCheck -check-prefix=CHK-BCLIB-DIR %s
 166
 167 /// Create a bogus bitcode library and find it with LIBRARY_PATH
 168 // RUN:   env LIBRARY_PATH=%S/Inputs/libomptarget/subdir %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 169 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 170 // RUN:   -fopenmp-relocatable-target -save-temps \
 171 // RUN:   %s 2>&1 | FileCheck -check-prefix=CHK-ENV-BCLIB %s
 172
 173 // CHK-BCLIB: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}libomptarget-nvptx-test.bc
 174 // CHK-BCLIB-DIR: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}libomptarget{{/|\\\\}}libomptarget-nvptx-sm_35.bc
 175 // CHK-ENV-BCLIB: clang{{.*}}-triple{{.*}}nvptx64-nvidia-cuda{{.*}}-mlink-builtin-bitcode{{.*}}subdir{{/|\\\\}}libomptarget-nvptx-sm_35.bc
 176 // CHK-BCLIB-NOT: {{error:|warning:}}
 177
 178 /// ###########################################################################
 179
 180 /// Check that the warning is thrown when the libomptarget bitcode library is not found.
 181 /// Libomptarget requires sm_35 or newer so an sm_35 bitcode library should never exist.
 182 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 183 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 184 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
 185 // RUN:   | FileCheck -check-prefix=CHK-BCLIB-WARN %s
 186
 187 // CHK-BCLIB-WARN: no library 'libomptarget-nvptx-sm_35.bc' found in the default clang lib directory or in LIBRARY_PATH; use '--libomptarget-nvptx-bc-path' to specify nvptx bitcode library
 188
 189 /// ###########################################################################
 190
 191 /// Check that the error is thrown when the libomptarget bitcode library does not exist.
 192 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 193 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_102/usr/local/cuda \
 194 // RUN:   --libomptarget-nvptx-bc-path=not-exist.bc \
 195 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
 196 // RUN:   | FileCheck -check-prefix=CHK-BCLIB-ERROR %s
 197
 198 // CHK-BCLIB-ERROR: bitcode library 'not-exist.bc' does not exist
 199
 200 /// ###########################################################################
 201
 202 /// Check that the error is thrown when CUDA 9.1 or lower version is used.
 203 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 204 // RUN:   -Xopenmp-target -march=sm_35 --cuda-path=%S/Inputs/CUDA_90/usr/local/cuda \
 205 // RUN:   -fopenmp-relocatable-target -save-temps %s 2>&1 \
 206 // RUN:   | FileCheck -check-prefix=CHK-CUDA-VERSION-ERROR %s
 207
 208 // CHK-CUDA-VERSION-ERROR: NVPTX target requires CUDA 9.2 or above; CUDA 9.0 detected
 209
 210 /// Check that debug info is emitted in dwarf-2
 211 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O1 --no-cuda-noopt-device-debug 2>&1 \
 212 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
 213 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 2>&1 \
 214 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
 215 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 --no-cuda-noopt-device-debug 2>&1 \
 216 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
 217 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g0 2>&1 \
 218 // RUN:   | FileCheck -check-prefix=NO_DEBUG %s
 219 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb0 -O3 --cuda-noopt-device-debug 2>&1 \
 220 // RUN:   | FileCheck -check-prefix=NO_DEBUG %s
 221 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -gline-directives-only 2>&1 \
 222 // RUN:   | FileCheck -check-prefix=DEBUG_DIRECTIVES %s
 223
 224 // DEBUG_DIRECTIVES-NOT: warning: debug
 225 // NO_DEBUG-NOT: warning: debug
 226 // NO_DEBUG: "-fopenmp-is-device"
 227 // NO_DEBUG-NOT: "-debug-info-kind=
 228 // NO_DEBUG: ptxas
 229 // DEBUG_DIRECTIVES: "-triple" "nvptx64-nvidia-cuda"
 230 // DEBUG_DIRECTIVES-SAME: "-debug-info-kind=line-directives-only"
 231 // DEBUG_DIRECTIVES-SAME: "-fopenmp-is-device"
 232 // DEBUG_DIRECTIVES: ptxas
 233 // DEBUG_DIRECTIVES: "-lineinfo"
 234 // NO_DEBUG-NOT: "-g"
 235 // NO_DEBUG: nvlink
 236 // NO_DEBUG-NOT: "-g"
 237
 238 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O0 --no-cuda-noopt-device-debug 2>&1 \
 239 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 240 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g 2>&1 \
 241 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 242 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O0 --cuda-noopt-device-debug 2>&1 \
 243 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 244 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g -O3 --cuda-noopt-device-debug 2>&1 \
 245 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 246 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g2 2>&1 \
 247 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 248 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb2 -O0 --cuda-noopt-device-debug 2>&1 \
 249 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 250 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -g3 -O3 --cuda-noopt-device-debug 2>&1 \
 251 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 252 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb3 -O2 --cuda-noopt-device-debug 2>&1 \
 253 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 254 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -gline-tables-only 2>&1 \
 255 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 256 // RUN:   %clang -### -fno-openmp-new-driver -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -ggdb1 -O2 --cuda-noopt-device-debug 2>&1 \
 257 // RUN:   | FileCheck -check-prefix=HAS_DEBUG %s
 258
 259 // HAS_DEBUG-NOT: warning: debug
 260 // HAS_DEBUG: "-triple" "nvptx64-nvidia-cuda"
 261 // HAS_DEBUG-SAME: "-debug-info-kind={{constructor|line-tables-only}}"
 262 // HAS_DEBUG-SAME: "-dwarf-version=2"
 263 // HAS_DEBUG-SAME: "-fopenmp-is-device"
 264 // HAS_DEBUG: ptxas
 265 // HAS_DEBUG-SAME: "-g"
 266 // HAS_DEBUG-SAME: "--dont-merge-basicblocks"
 267 // HAS_DEBUG-SAME: "--return-at-end"
 268 // HAS_DEBUG: nvlink
 269 // HAS_DEBUG-SAME: "-g"
 270
 271 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-mode 2>&1 \
 272 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
 273 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-mode -fopenmp-cuda-mode 2>&1 \
 274 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
 275 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-mode 2>&1 \
 276 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
 277 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-mode -fopenmp-cuda-mode 2>&1 \
 278 // RUN:   | FileCheck -check-prefix=CUDA_MODE %s
 279 // CUDA_MODE: "-cc1"{{.*}}"-triple" "{{nvptx64-nvidia-cuda|amdgcn-amd-amdhsa}}"
 280 // CUDA_MODE-SAME: "-fopenmp-cuda-mode"
 281 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-mode 2>&1 \
 282 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
 283 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-mode -fno-openmp-cuda-mode 2>&1 \
 284 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
 285 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-mode 2>&1 \
 286 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
 287 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-mode -fno-openmp-cuda-mode 2>&1 \
 288 // RUN:   | FileCheck -check-prefix=NO_CUDA_MODE %s
 289 // NO_CUDA_MODE-NOT: "-{{fno-|f}}openmp-cuda-mode"
 290
 291 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-force-full-runtime 2>&1 \
 292 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
 293 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-force-full-runtime -fopenmp-cuda-force-full-runtime 2>&1 \
 294 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
 295 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-force-full-runtime 2>&1 \
 296 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
 297 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-force-full-runtime -fopenmp-cuda-force-full-runtime 2>&1 \
 298 // RUN:   | FileCheck -check-prefix=FULL_RUNTIME %s
 299 // FULL_RUNTIME: "-cc1"{{.*}}"-triple" "{{nvptx64-nvidia-cuda|amdgcn-amd-amdhsa}}"
 300 // FULL_RUNTIME-SAME: "-fopenmp-cuda-force-full-runtime"
 301 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fno-openmp-cuda-force-full-runtime 2>&1 \
 302 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
 303 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-force-full-runtime -fno-openmp-cuda-force-full-runtime 2>&1 \
 304 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
 305 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fno-openmp-cuda-force-full-runtime 2>&1 \
 306 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
 307 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target -march=gfx906 %s -fopenmp-cuda-force-full-runtime -fno-openmp-cuda-force-full-runtime 2>&1 \
 308 // RUN:   | FileCheck -check-prefix=NO_FULL_RUNTIME %s
 309 // NO_FULL_RUNTIME-NOT: "-{{fno-|f}}openmp-cuda-force-full-runtime"
 310
 311 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target -march=sm_60 %s -fopenmp-cuda-teams-reduction-recs-num=2048 2>&1 \
 312 // RUN:   | FileCheck -check-prefix=CUDA_RED_RECS %s
 313 // CUDA_RED_RECS: "-cc1"{{.*}}"-triple" "nvptx64-nvidia-cuda"
 314 // CUDA_RED_RECS-SAME: "-fopenmp-cuda-teams-reduction-recs-num=2048"
 315
 316 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda %s 2>&1 \
 317 // RUN:   | FileCheck -check-prefix=OPENMP_NVPTX_WRAPPERS %s
 318 // OPENMP_NVPTX_WRAPPERS: "-cc1"{{.*}}"-triple" "nvptx64-nvidia-cuda"
 319 // OPENMP_NVPTX_WRAPPERS-SAME: "-internal-isystem" "{{.*}}openmp_wrappers"
 320
 321 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda \
 322 // RUN:          -save-temps -ccc-print-bindings %s -o openmp-offload-gpu 2>&1 \
 323 // RUN:   | FileCheck -check-prefix=SAVE_TEMPS_NAMES %s
 324
 325 // SAVE_TEMPS_NAMES-NOT: "GNU::Linker"{{.*}}["[[SAVE_TEMPS_INPUT1:.*\.o]]", "[[SAVE_TEMPS_INPUT1]]"]
 326
 327 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64 -Xopenmp-target=nvptx64 -march=sm_35 \
 328 // RUN:          -save-temps %s -o openmp-offload-gpu 2>&1 \
 329 // RUN:   | FileCheck -check-prefix=TRIPLE %s
 330
 331 // TRIPLE: "-triple" "nvptx64-nvidia-cuda"
 332 // TRIPLE: "-target-cpu" "sm_35"