1from __future__ import absolute_import
2import json
3import math
4import os
5import shlex
6import subprocess
7import sys
8
9import lit.Test
10import lit.TestRunner
11import lit.util
12from .base import TestFormat
13
14kIsWindows = sys.platform in ['win32', 'cygwin']
15
16class GoogleTest(TestFormat):
17    def __init__(self, test_sub_dirs, test_suffix, run_under = []):
18        self.test_sub_dirs = str(test_sub_dirs).split(';')
19
20        # On Windows, assume tests will also end in '.exe'.
21        exe_suffix = str(test_suffix)
22        if kIsWindows:
23            exe_suffix += '.exe'
24
25        # Also check for .py files for testing purposes.
26        self.test_suffixes = {exe_suffix, test_suffix + '.py'}
27        self.run_under = run_under
28
29    def get_num_tests(self, path, litConfig, localConfig):
30        list_test_cmd = self.prepareCmd(
31            [path, '--gtest_list_tests', '--gtest_filter=-*DISABLED_*'])
32        try:
33            out = subprocess.check_output(list_test_cmd,
34                                          env=localConfig.environment)
35        except subprocess.CalledProcessError as exc:
36            litConfig.warning(
37                "unable to discover google-tests in %r: %s. Process output: %s"
38                % (path, sys.exc_info()[1], exc.output))
39            return None
40        return sum(
41            map(lambda line: lit.util.to_string(line).startswith('  '),
42                out.splitlines(False)))
43
44    def getTestsInDirectory(self, testSuite, path_in_suite, litConfig,
45                            localConfig):
46        init_shard_size = 512  # number of tests in a shard
47        core_count = lit.util.usable_core_count()
48        source_path = testSuite.getSourcePath(path_in_suite)
49        for subdir in self.test_sub_dirs:
50            dir_path = os.path.join(source_path, subdir)
51            if not os.path.isdir(dir_path):
52                continue
53            for fn in lit.util.listdir_files(dir_path,
54                                             suffixes=self.test_suffixes):
55                # Discover the tests in this executable.
56                execpath = os.path.join(source_path, subdir, fn)
57                num_tests = self.get_num_tests(execpath, litConfig,
58                                               localConfig)
59                if num_tests is not None:
60                    # Compute the number of shards.
61                    shard_size = init_shard_size
62                    nshard = int(math.ceil(num_tests / shard_size))
63                    while nshard < core_count and shard_size > 1:
64                        shard_size = shard_size // 2
65                        nshard = int(math.ceil(num_tests / shard_size))
66
67                    # Create one lit test for each shard.
68                    for idx in range(nshard):
69                        testPath = path_in_suite + (subdir, fn, str(idx),
70                                                    str(nshard))
71                        json_file = '-'.join([
72                            execpath, testSuite.config.name,
73                            str(os.getpid()),
74                            str(idx),
75                            str(nshard)
76                        ]) + '.json'
77                        yield lit.Test.Test(testSuite,
78                                            testPath,
79                                            localConfig,
80                                            file_path=execpath,
81                                            gtest_json_file=json_file)
82                else:
83                    # This doesn't look like a valid gtest file.  This can
84                    # have a number of causes, none of them good.  For
85                    # instance, we could have created a broken executable.
86                    # Alternatively, someone has cruft in their test
87                    # directory.  If we don't return a test here, then no
88                    # failures will get reported, so return a dummy test name
89                    # so that the failure is reported later.
90                    testPath = path_in_suite + (
91                        subdir, fn, 'failed_to_discover_tests_from_gtest')
92                    yield lit.Test.Test(testSuite,
93                                        testPath,
94                                        localConfig,
95                                        file_path=execpath)
96
97    def execute(self, test, litConfig):
98        if test.gtest_json_file is None:
99            return lit.Test.FAIL, ''
100
101        testPath,testName = os.path.split(test.getSourcePath())
102        while not os.path.exists(testPath):
103            # Handle GTest parametrized and typed tests, whose name includes
104            # some '/'s.
105            testPath, namePrefix = os.path.split(testPath)
106            testName = namePrefix + '/' + testName
107
108        testName,total_shards = os.path.split(testName)
109        testName,shard_idx = os.path.split(testName)
110        from lit.cl_arguments import TestOrder
111        use_shuffle = TestOrder(litConfig.order) == TestOrder.RANDOM
112        shard_env = {
113            'GTEST_OUTPUT': 'json:' + test.gtest_json_file,
114            'GTEST_SHUFFLE': '1' if use_shuffle else '0',
115            'GTEST_TOTAL_SHARDS': total_shards,
116            'GTEST_SHARD_INDEX': shard_idx
117        }
118        test.config.environment.update(shard_env)
119
120        cmd = [testPath]
121        cmd = self.prepareCmd(cmd)
122        if litConfig.useValgrind:
123            cmd = litConfig.valgrindArgs + cmd
124
125        if litConfig.noExecute:
126            return lit.Test.PASS, ''
127
128        def get_shard_header(shard_env):
129            shard_envs = ' '.join([k + '=' + v for k, v in shard_env.items()])
130            return f"Script(shard):\n--\n%s %s\n--\n" % (shard_envs, ' '.join(cmd))
131
132        shard_header = get_shard_header(shard_env)
133
134        try:
135            out, _, exitCode = lit.util.executeCommand(
136                cmd, env=test.config.environment,
137                timeout=litConfig.maxIndividualTestTime, redirect_stderr=True)
138        except lit.util.ExecuteCommandTimeoutException as e:
139            stream_msg = f"\n{e.out}\n--\nexit: {e.exitCode}\n--\n"
140            return (lit.Test.TIMEOUT, f'{shard_header}{stream_msg}Reached '
141                    f'timeout of {litConfig.maxIndividualTestTime} seconds')
142
143        if not os.path.exists(test.gtest_json_file):
144            errmsg = f"shard JSON output does not exist: %s" % (
145                test.gtest_json_file)
146            stream_msg = f"\n{out}\n--\nexit: {exitCode}\n--\n"
147            return lit.Test.FAIL, shard_header + stream_msg + errmsg
148
149        if exitCode == 0:
150            return lit.Test.PASS, ''
151
152        def get_test_stdout(test_name):
153            res = []
154            header = f'[ RUN      ] ' + test_name
155            footer = f'[  FAILED  ] ' + test_name
156            in_range = False
157            for l in out.splitlines():
158                if l.startswith(header):
159                    in_range = True
160                elif l.startswith(footer):
161                    return f'' if len(res) == 0 else '\n'.join(res)
162                elif in_range:
163                    res.append(l)
164            assert False, f'gtest did not report the result for ' + test_name
165
166        found_failed_test = False
167
168        with open(test.gtest_json_file, encoding='utf-8') as f:
169            jf = json.load(f)
170
171            if use_shuffle:
172                shard_env['GTEST_RANDOM_SEED'] = str(jf['random_seed'])
173            output = get_shard_header(shard_env) + '\n'
174
175            for testcase in jf['testsuites']:
176                for testinfo in testcase['testsuite']:
177                    result = testinfo['result']
178                    if result == 'SUPPRESSED' or result == 'SKIPPED':
179                        continue
180                    testname = testcase['name'] + '.' + testinfo['name']
181                    header = f"Script:\n--\n%s --gtest_filter=%s\n--\n" % (
182                        ' '.join(cmd), testname)
183                    if 'failures' in testinfo:
184                        found_failed_test = True
185                        output += header
186                        test_out = get_test_stdout(testname)
187                        if test_out:
188                            output += test_out + '\n\n'
189                        for fail in testinfo['failures']:
190                            output += fail['failure'] + '\n'
191                        output += '\n'
192                    elif result != 'COMPLETED':
193                        output += header
194                        output += 'unresolved test result\n'
195
196        # In some situations, like running tests with sanitizers, all test passes but
197        # the shard could still fail due to memory issues.
198        if not found_failed_test:
199            output += f"\n{out}\n--\nexit: {exitCode}\n--\n"
200
201        return lit.Test.FAIL, output
202
203    def prepareCmd(self, cmd):
204        """Insert interpreter if needed.
205
206        It inserts the python exe into the command if cmd[0] ends in .py or caller
207        specified run_under.
208        We cannot rely on the system to interpret shebang lines for us on
209        Windows, so add the python executable to the command if this is a .py
210        script.
211        """
212        if cmd[0].endswith('.py'):
213            cmd = [sys.executable] + cmd
214        if self.run_under:
215            if isinstance(self.run_under, list):
216                cmd = self.run_under + cmd
217            else:
218                cmd = shlex.split(self.run_under) + cmd
219        return cmd
220
221    @staticmethod
222    def post_process_shard_results(selected_tests, discovered_tests):
223        def remove_gtest(tests):
224            return [t for t in tests if t.gtest_json_file is None]
225
226        discovered_tests = remove_gtest(discovered_tests)
227        gtests = [t for t in selected_tests if t.gtest_json_file]
228        selected_tests = remove_gtest(selected_tests)
229        for test in gtests:
230            # In case gtest has bugs such that no JSON file was emitted.
231            if not os.path.exists(test.gtest_json_file):
232                selected_tests.append(test)
233                discovered_tests.append(test)
234                continue
235
236            start_time = test.result.start or 0.0
237
238            has_failure_in_shard = False
239
240            # Load json file to retrieve results.
241            with open(test.gtest_json_file, encoding='utf-8') as f:
242                testsuites = json.load(f)['testsuites']
243                for testcase in testsuites:
244                    for testinfo in testcase['testsuite']:
245                        # Ignore disabled tests.
246                        if testinfo['result'] == 'SUPPRESSED':
247                            continue
248
249                        testPath = test.path_in_suite[:-2] + (testcase['name'],
250                                                              testinfo['name'])
251                        subtest = lit.Test.Test(test.suite, testPath,
252                                                test.config, test.file_path)
253
254                        testname = testcase['name'] + '.' + testinfo['name']
255                        header = f"Script:\n--\n%s --gtest_filter=%s\n--\n" % (
256                            test.file_path, testname)
257
258                        output = ''
259                        if testinfo['result'] == 'SKIPPED':
260                            returnCode = lit.Test.SKIPPED
261                        elif 'failures' in testinfo:
262                            has_failure_in_shard = True
263                            returnCode = lit.Test.FAIL
264                            output = header
265                            for fail in testinfo['failures']:
266                                output += fail['failure'] + '\n'
267                        elif testinfo['result'] == 'COMPLETED':
268                            returnCode = lit.Test.PASS
269                        else:
270                            returnCode = lit.Test.UNRESOLVED
271                            output = header + 'unresolved test result\n'
272
273                        elapsed_time = float(testinfo['time'][:-1])
274                        res = lit.Test.Result(returnCode, output, elapsed_time)
275                        res.pid = test.result.pid or 0
276                        res.start = start_time
277                        start_time = start_time + elapsed_time
278                        subtest.setResult(res)
279
280                        selected_tests.append(subtest)
281                        discovered_tests.append(subtest)
282            os.remove(test.gtest_json_file)
283
284            if not has_failure_in_shard and test.isFailure():
285                selected_tests.append(test)
286                discovered_tests.append(test)
287
288        return selected_tests, discovered_tests
289