1"""
2Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
3See https://llvm.org/LICENSE.txt for license information.
4SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
5
6Provides the LLDBTestResult class, which holds information about progress
7and results of a single test run.
8"""
9
10from __future__ import absolute_import
11from __future__ import print_function
12
13# System modules
14import inspect
15import os
16
17# Third-party modules
18import unittest2
19
20# LLDB Modules
21from . import configuration
22from lldbsuite.test_event.event_builder import EventBuilder
23from lldbsuite.test_event import build_exception
24
25
26class LLDBTestResult(unittest2.TextTestResult):
27    """
28    Enforce a singleton pattern to allow introspection of test progress.
29
30    Overwrite addError(), addFailure(), and addExpectedFailure() methods
31    to enable each test instance to track its failure/error status.  It
32    is used in the LLDB test framework to emit detailed trace messages
33    to a log file for easier human inspection of test failures/errors.
34    """
35    __singleton__ = None
36    __ignore_singleton__ = False
37
38    @staticmethod
39    def getTerminalSize():
40        import os
41        env = os.environ
42
43        def ioctl_GWINSZ(fd):
44            try:
45                import fcntl
46                import termios
47                import struct
48                cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
49                                                     '1234'))
50            except:
51                return
52            return cr
53        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
54        if not cr:
55            try:
56                fd = os.open(os.ctermid(), os.O_RDONLY)
57                cr = ioctl_GWINSZ(fd)
58                os.close(fd)
59            except:
60                pass
61        if not cr:
62            cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
63        return int(cr[1]), int(cr[0])
64
65    def __init__(self, *args):
66        if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
67            raise Exception("LLDBTestResult instantiated more than once")
68        super(LLDBTestResult, self).__init__(*args)
69        LLDBTestResult.__singleton__ = self
70        # Now put this singleton into the lldb module namespace.
71        configuration.test_result = self
72        # Computes the format string for displaying the counter.
73        counterWidth = len(str(configuration.suite.countTestCases()))
74        self.fmt = "%" + str(counterWidth) + "d: "
75        self.indentation = ' ' * (counterWidth + 2)
76        # This counts from 1 .. suite.countTestCases().
77        self.counter = 0
78        (width, height) = LLDBTestResult.getTerminalSize()
79        self.results_formatter = configuration.results_formatter_object
80
81    def _config_string(self, test):
82        compiler = getattr(test, "getCompiler", None)
83        arch = getattr(test, "getArchitecture", None)
84        return "%s-%s" % (compiler() if compiler else "",
85                          arch() if arch else "")
86
87    def _exc_info_to_string(self, err, test):
88        """Overrides superclass TestResult's method in order to append
89        our test config info string to the exception info string."""
90        if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
91            return '%sConfig=%s-%s' % (super(LLDBTestResult,
92                                             self)._exc_info_to_string(err,
93                                                                       test),
94                                       test.getArchitecture(),
95                                       test.getCompiler())
96        else:
97            return super(LLDBTestResult, self)._exc_info_to_string(err, test)
98
99    def getDescription(self, test):
100        doc_first_line = test.shortDescription()
101        if self.descriptions and doc_first_line:
102            return '\n'.join((str(test), self.indentation + doc_first_line))
103        else:
104            return str(test)
105
106    @staticmethod
107    def _getFileBasedCategories(test):
108        """
109        Returns the list of categories to which this test case belongs by
110        looking for a ".categories" file. We start at the folder the test is in
111        an traverse the hierarchy upwards - we guarantee a .categories to exist
112        at the top level directory so we do not end up looping endlessly.
113        """
114        import inspect
115        import os.path
116        folder = inspect.getfile(test.__class__)
117        folder = os.path.dirname(folder)
118        while folder != '/':
119            categories_file_name = os.path.join(folder, ".categories")
120            if os.path.exists(categories_file_name):
121                categories_file = open(categories_file_name, 'r')
122                categories = categories_file.readline()
123                categories_file.close()
124                categories = str.replace(categories, '\n', '')
125                categories = str.replace(categories, '\r', '')
126                return categories.split(',')
127            else:
128                folder = os.path.dirname(folder)
129                continue
130
131
132    def getCategoriesForTest(self, test):
133        """
134        Gets all the categories for the currently running test method in test case
135        """
136        test_categories = []
137        test_method = getattr(test, test._testMethodName)
138        if test_method is not None and hasattr(test_method, "categories"):
139            test_categories.extend(test_method.categories)
140
141        test_categories.extend(self._getFileBasedCategories(test))
142
143        return test_categories
144
145    def hardMarkAsSkipped(self, test):
146        getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
147        getattr(
148            test,
149            test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
150
151    def checkExclusion(self, exclusion_list, name):
152        if exclusion_list:
153            import re
154            for item in exclusion_list:
155                if re.search(item, name):
156                    return True
157        return False
158
159    def startTest(self, test):
160        if configuration.shouldSkipBecauseOfCategories(
161                self.getCategoriesForTest(test)):
162            self.hardMarkAsSkipped(test)
163        if self.checkExclusion(
164                configuration.skip_tests, test.id()):
165            self.hardMarkAsSkipped(test)
166
167        self.counter += 1
168        test.test_number = self.counter
169        if self.showAll:
170            self.stream.write(self.fmt % self.counter)
171        super(LLDBTestResult, self).startTest(test)
172        if self.results_formatter:
173            self.results_formatter.handle_event(
174                EventBuilder.event_for_start(test))
175
176    def addSuccess(self, test):
177        if self.checkExclusion(
178                configuration.xfail_tests, test.id()):
179            self.addUnexpectedSuccess(test, None)
180            return
181
182        super(LLDBTestResult, self).addSuccess(test)
183        if configuration.parsable:
184            self.stream.write(
185                "PASS: LLDB (%s) :: %s\n" %
186                (self._config_string(test), str(test)))
187        if self.results_formatter:
188            self.results_formatter.handle_event(
189                EventBuilder.event_for_success(test))
190
191    def _isBuildError(self, err_tuple):
192        exception = err_tuple[1]
193        return isinstance(exception, build_exception.BuildError)
194
195    def _getTestPath(self, test):
196        if test is None:
197            return ""
198        elif hasattr(test, "test_filename"):
199            return test.test_filename
200        else:
201            return inspect.getsourcefile(test.__class__)
202
203    def _saveBuildErrorTuple(self, test, err):
204        # Adjust the error description so it prints the build command and build error
205        # rather than an uninformative Python backtrace.
206        build_error = err[1]
207        error_description = "{}\nTest Directory:\n{}".format(
208            str(build_error),
209            os.path.dirname(self._getTestPath(test)))
210        self.errors.append((test, error_description))
211        self._mirrorOutput = True
212
213    def addError(self, test, err):
214        configuration.sdir_has_content = True
215        if self._isBuildError(err):
216            self._saveBuildErrorTuple(test, err)
217        else:
218            super(LLDBTestResult, self).addError(test, err)
219
220        method = getattr(test, "markError", None)
221        if method:
222            method()
223        if configuration.parsable:
224            self.stream.write(
225                "FAIL: LLDB (%s) :: %s\n" %
226                (self._config_string(test), str(test)))
227        if self.results_formatter:
228            # Handle build errors as a separate event type
229            if self._isBuildError(err):
230                error_event = EventBuilder.event_for_build_error(test, err)
231            else:
232                error_event = EventBuilder.event_for_error(test, err)
233            self.results_formatter.handle_event(error_event)
234
235    def addCleanupError(self, test, err):
236        configuration.sdir_has_content = True
237        super(LLDBTestResult, self).addCleanupError(test, err)
238        method = getattr(test, "markCleanupError", None)
239        if method:
240            method()
241        if configuration.parsable:
242            self.stream.write(
243                "CLEANUP ERROR: LLDB (%s) :: %s\n" %
244                (self._config_string(test), str(test)))
245        if self.results_formatter:
246            self.results_formatter.handle_event(
247                EventBuilder.event_for_cleanup_error(
248                    test, err))
249
250    def addFailure(self, test, err):
251        if self.checkExclusion(
252                configuration.xfail_tests, test.id()):
253            self.addExpectedFailure(test, err, None)
254            return
255
256        configuration.sdir_has_content = True
257        super(LLDBTestResult, self).addFailure(test, err)
258        method = getattr(test, "markFailure", None)
259        if method:
260            method()
261        if configuration.parsable:
262            self.stream.write(
263                "FAIL: LLDB (%s) :: %s\n" %
264                (self._config_string(test), str(test)))
265        if configuration.useCategories:
266            test_categories = self.getCategoriesForTest(test)
267            for category in test_categories:
268                if category in configuration.failuresPerCategory:
269                    configuration.failuresPerCategory[
270                        category] = configuration.failuresPerCategory[category] + 1
271                else:
272                    configuration.failuresPerCategory[category] = 1
273        if self.results_formatter:
274            self.results_formatter.handle_event(
275                EventBuilder.event_for_failure(test, err))
276
277    def addExpectedFailure(self, test, err, bugnumber):
278        configuration.sdir_has_content = True
279        super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
280        method = getattr(test, "markExpectedFailure", None)
281        if method:
282            method(err, bugnumber)
283        if configuration.parsable:
284            self.stream.write(
285                "XFAIL: LLDB (%s) :: %s\n" %
286                (self._config_string(test), str(test)))
287        if self.results_formatter:
288            self.results_formatter.handle_event(
289                EventBuilder.event_for_expected_failure(
290                    test, err, bugnumber))
291
292    def addSkip(self, test, reason):
293        configuration.sdir_has_content = True
294        super(LLDBTestResult, self).addSkip(test, reason)
295        method = getattr(test, "markSkippedTest", None)
296        if method:
297            method()
298        if configuration.parsable:
299            self.stream.write(
300                "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
301                (self._config_string(test), str(test), reason))
302        if self.results_formatter:
303            self.results_formatter.handle_event(
304                EventBuilder.event_for_skip(test, reason))
305
306    def addUnexpectedSuccess(self, test, bugnumber):
307        configuration.sdir_has_content = True
308        super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
309        method = getattr(test, "markUnexpectedSuccess", None)
310        if method:
311            method(bugnumber)
312        if configuration.parsable:
313            self.stream.write(
314                "XPASS: LLDB (%s) :: %s\n" %
315                (self._config_string(test), str(test)))
316        if self.results_formatter:
317            self.results_formatter.handle_event(
318                EventBuilder.event_for_unexpected_success(
319                    test, bugnumber))
320