1#===----------------------------------------------------------------------===## 2# 3# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4# See https://llvm.org/LICENSE.txt for license information. 5# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6# 7#===----------------------------------------------------------------------===## 8 9import lit 10import lit.formats 11import os 12import pipes 13import re 14import shutil 15 16def _getTempPaths(test): 17 """ 18 Return the values to use for the %T and %t substitutions, respectively. 19 20 The difference between this and Lit's default behavior is that we guarantee 21 that %T is a path unique to the test being run. 22 """ 23 tmpDir, _ = lit.TestRunner.getTempPaths(test) 24 _, testName = os.path.split(test.getExecPath()) 25 tmpDir = os.path.join(tmpDir, testName + '.dir') 26 tmpBase = os.path.join(tmpDir, 't') 27 return tmpDir, tmpBase 28 29def _checkBaseSubstitutions(substitutions): 30 substitutions = [s for (s, _) in substitutions] 31 for s in ['%{cxx}', '%{compile_flags}', '%{link_flags}', '%{flags}', '%{exec}']: 32 assert s in substitutions, "Required substitution {} was not provided".format(s) 33 34def parseScript(test, preamble): 35 """ 36 Extract the script from a test, with substitutions applied. 37 38 Returns a list of commands ready to be executed. 39 40 - test 41 The lit.Test to parse. 42 43 - preamble 44 A list of commands to perform before any command in the test. 45 These commands can contain unexpanded substitutions, but they 46 must not be of the form 'RUN:' -- they must be proper commands 47 once substituted. 48 """ 49 # Get the default substitutions 50 tmpDir, tmpBase = _getTempPaths(test) 51 substitutions = lit.TestRunner.getDefaultSubstitutions(test, tmpDir, tmpBase) 52 53 # Check base substitutions and add the %{build} and %{run} convenience substitutions 54 _checkBaseSubstitutions(substitutions) 55 substitutions.append(('%{build}', '%{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe')) 56 substitutions.append(('%{run}', '%{exec} %t.exe')) 57 58 # Parse the test file, including custom directives 59 additionalCompileFlags = [] 60 fileDependencies = [] 61 parsers = [ 62 lit.TestRunner.IntegratedTestKeywordParser('FILE_DEPENDENCIES:', 63 lit.TestRunner.ParserKind.LIST, 64 initial_value=fileDependencies), 65 lit.TestRunner.IntegratedTestKeywordParser('ADDITIONAL_COMPILE_FLAGS:', 66 lit.TestRunner.ParserKind.LIST, 67 initial_value=additionalCompileFlags) 68 ] 69 70 scriptInTest = lit.TestRunner.parseIntegratedTestScript(test, additional_parsers=parsers, 71 require_script=not preamble) 72 if isinstance(scriptInTest, lit.Test.Result): 73 return scriptInTest 74 75 script = [] 76 77 # For each file dependency in FILE_DEPENDENCIES, inject a command to copy 78 # that file to the execution directory. Execute the copy from %S to allow 79 # relative paths from the test directory. 80 for dep in fileDependencies: 81 script += ['%dbg(SETUP) cd %S && cp {} %T'.format(dep)] 82 script += preamble 83 script += scriptInTest 84 85 # Add compile flags specified with ADDITIONAL_COMPILE_FLAGS. 86 substitutions = [(s, x + ' ' + ' '.join(additionalCompileFlags)) if s == '%{compile_flags}' 87 else (s, x) for (s, x) in substitutions] 88 89 # Perform substitutions in the script itself. 90 script = lit.TestRunner.applySubstitutions(script, substitutions, 91 recursion_limit=test.config.recursiveExpansionLimit) 92 93 return script 94 95 96class CxxStandardLibraryTest(lit.formats.TestFormat): 97 """ 98 Lit test format for the C++ Standard Library conformance test suite. 99 100 This test format is based on top of the ShTest format -- it basically 101 creates a shell script performing the right operations (compile/link/run) 102 based on the extension of the test file it encounters. It supports files 103 with the following extensions: 104 105 FOO.pass.cpp - Compiles, links and runs successfully 106 FOO.pass.mm - Same as .pass.cpp, but for Objective-C++ 107 108 FOO.compile.pass.cpp - Compiles successfully, link and run not attempted 109 FOO.compile.fail.cpp - Does not compile successfully 110 111 FOO.link.pass.cpp - Compiles and links successfully, run not attempted 112 FOO.link.fail.cpp - Compiles successfully, but fails to link 113 114 FOO.sh.<anything> - A builtin Lit Shell test 115 116 FOO.verify.cpp - Compiles with clang-verify. This type of test is 117 automatically marked as UNSUPPORTED if the compiler 118 does not support Clang-verify. 119 120 FOO.fail.cpp - Compiled with clang-verify if clang-verify is 121 supported, and equivalent to a .compile.fail.cpp 122 test otherwise. This is supported only for backwards 123 compatibility with the test suite. 124 125 126 Substitution requirements 127 =============================== 128 The test format operates by assuming that each test's configuration provides 129 the following substitutions, which it will reuse in the shell scripts it 130 constructs: 131 %{cxx} - A command that can be used to invoke the compiler 132 %{compile_flags} - Flags to use when compiling a test case 133 %{link_flags} - Flags to use when linking a test case 134 %{flags} - Flags to use either when compiling or linking a test case 135 %{exec} - A command to prefix the execution of executables 136 137 Note that when building an executable (as opposed to only compiling a source 138 file), all three of %{flags}, %{compile_flags} and %{link_flags} will be used 139 in the same command line. In other words, the test format doesn't perform 140 separate compilation and linking steps in this case. 141 142 143 Additional supported directives 144 =============================== 145 In addition to everything that's supported in Lit ShTests, this test format 146 also understands the following directives inside test files: 147 148 // FILE_DEPENDENCIES: file, directory, /path/to/file 149 150 This directive expresses that the test requires the provided files 151 or directories in order to run. An example is a test that requires 152 some test input stored in a data file. When a test file contains 153 such a directive, this test format will collect them and copy them 154 to the directory represented by %T. The intent is that %T contains 155 all the inputs necessary to run the test, such that e.g. execution 156 on a remote host can be done by simply copying %T to the host. 157 158 // ADDITIONAL_COMPILE_FLAGS: flag1, flag2, flag3 159 160 This directive will cause the provided flags to be added to the 161 %{compile_flags} substitution for the test that contains it. This 162 allows adding special compilation flags without having to use a 163 .sh.cpp test, which would be more powerful but perhaps overkill. 164 165 166 Additional provided substitutions and features 167 ============================================== 168 The test format will define the following substitutions for use inside tests: 169 170 %{build} 171 Expands to a command-line that builds the current source 172 file with the %{flags}, %{compile_flags} and %{link_flags} 173 substitutions, and that produces an executable named %t.exe. 174 175 %{run} 176 Equivalent to `%{exec} %t.exe`. This is intended to be used 177 in conjunction with the %{build} substitution. 178 """ 179 def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig): 180 SUPPORTED_SUFFIXES = ['[.]pass[.]cpp$', '[.]pass[.]mm$', 181 '[.]compile[.]pass[.]cpp$', '[.]compile[.]fail[.]cpp$', 182 '[.]link[.]pass[.]cpp$', '[.]link[.]fail[.]cpp$', 183 '[.]sh[.][^.]+$', 184 '[.]verify[.]cpp$', 185 '[.]fail[.]cpp$'] 186 sourcePath = testSuite.getSourcePath(pathInSuite) 187 for filename in os.listdir(sourcePath): 188 # Ignore dot files and excluded tests. 189 if filename.startswith('.') or filename in localConfig.excludes: 190 continue 191 192 filepath = os.path.join(sourcePath, filename) 193 if not os.path.isdir(filepath): 194 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]): 195 yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig) 196 197 def execute(self, test, litConfig): 198 VERIFY_FLAGS = '-Xclang -verify -Xclang -verify-ignore-unexpected=note -ferror-limit=0' 199 supportsVerify = 'verify-support' in test.config.available_features 200 filename = test.path_in_suite[-1] 201 202 if re.search('[.]sh[.][^.]+$', filename): 203 steps = [ ] # The steps are already in the script 204 return self._executeShTest(test, litConfig, steps) 205 elif filename.endswith('.compile.pass.cpp'): 206 steps = [ 207 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 208 ] 209 return self._executeShTest(test, litConfig, steps) 210 elif filename.endswith('.compile.fail.cpp'): 211 steps = [ 212 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 213 ] 214 return self._executeShTest(test, litConfig, steps) 215 elif filename.endswith('.link.pass.cpp'): 216 steps = [ 217 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe" 218 ] 219 return self._executeShTest(test, litConfig, steps) 220 elif filename.endswith('.link.fail.cpp'): 221 steps = [ 222 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} -c -o %t.o", 223 "%dbg(LINKED WITH) ! %{cxx} %t.o %{flags} %{link_flags} -o %t.exe" 224 ] 225 return self._executeShTest(test, litConfig, steps) 226 elif filename.endswith('.verify.cpp'): 227 if not supportsVerify: 228 return lit.Test.Result(lit.Test.UNSUPPORTED, 229 "Test {} requires support for Clang-verify, which isn't supported by the compiler".format(test.getFullName())) 230 steps = [ 231 # Note: Use -Wno-error to make sure all diagnostics are not treated as errors, 232 # which doesn't make sense for clang-verify tests. 233 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 234 ] 235 return self._executeShTest(test, litConfig, steps) 236 # Make sure to check these ones last, since they will match other 237 # suffixes above too. 238 elif filename.endswith('.pass.cpp') or filename.endswith('.pass.mm'): 239 steps = [ 240 "%dbg(COMPILED WITH) %{cxx} %s %{flags} %{compile_flags} %{link_flags} -o %t.exe", 241 "%dbg(EXECUTED AS) %{exec} %t.exe" 242 ] 243 return self._executeShTest(test, litConfig, steps) 244 # This is like a .verify.cpp test when clang-verify is supported, 245 # otherwise it's like a .compile.fail.cpp test. This is only provided 246 # for backwards compatibility with the test suite. 247 elif filename.endswith('.fail.cpp'): 248 if supportsVerify: 249 steps = [ 250 "%dbg(COMPILED WITH) %{{cxx}} %s %{{flags}} %{{compile_flags}} -fsyntax-only -Wno-error {}".format(VERIFY_FLAGS) 251 ] 252 else: 253 steps = [ 254 "%dbg(COMPILED WITH) ! %{cxx} %s %{flags} %{compile_flags} -fsyntax-only" 255 ] 256 return self._executeShTest(test, litConfig, steps) 257 else: 258 return lit.Test.Result(lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)) 259 260 # Utility function to add compile flags in lit.local.cfg files. 261 def addCompileFlags(self, config, *flags): 262 string = ' '.join(flags) 263 config.substitutions = [(s, x + ' ' + string) if s == '%{compile_flags}' else (s, x) for (s, x) in config.substitutions] 264 265 def _executeShTest(self, test, litConfig, steps): 266 if test.config.unsupported: 267 return lit.Test.Result(lit.Test.UNSUPPORTED, 'Test is unsupported') 268 269 script = parseScript(test, steps) 270 if isinstance(script, lit.Test.Result): 271 return script 272 273 if litConfig.noExecute: 274 return lit.Test.Result(lit.Test.XFAIL if test.isExpectedToFail() else lit.Test.PASS) 275 else: 276 _, tmpBase = _getTempPaths(test) 277 useExternalSh = False 278 return lit.TestRunner._runShTest(test, litConfig, useExternalSh, script, tmpBase) 279