1"""
2                     The LLVM Compiler Infrastructure
3
4This file is distributed under the University of Illinois Open Source
5License. See LICENSE.TXT for details.
6
7Provides the LLDBTestResult class, which holds information about progress
8and results of a single test run.
9"""
10
11from __future__ import absolute_import
12from __future__ import print_function
13
14# System modules
15import inspect
16import os
17
18# Third-party modules
19import unittest2
20
21# LLDB Modules
22from . import configuration
23from lldbsuite.test_event.event_builder import EventBuilder
24from lldbsuite.test_event import build_exception
25
26
27class LLDBTestResult(unittest2.TextTestResult):
28    """
29    Enforce a singleton pattern to allow introspection of test progress.
30
31    Overwrite addError(), addFailure(), and addExpectedFailure() methods
32    to enable each test instance to track its failure/error status.  It
33    is used in the LLDB test framework to emit detailed trace messages
34    to a log file for easier human inspection of test failures/errors.
35    """
36    __singleton__ = None
37    __ignore_singleton__ = False
38
39    @staticmethod
40    def getTerminalSize():
41        import os
42        env = os.environ
43
44        def ioctl_GWINSZ(fd):
45            try:
46                import fcntl
47                import termios
48                import struct
49                import os
50                cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ,
51                                                     '1234'))
52            except:
53                return
54            return cr
55        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
56        if not cr:
57            try:
58                fd = os.open(os.ctermid(), os.O_RDONLY)
59                cr = ioctl_GWINSZ(fd)
60                os.close(fd)
61            except:
62                pass
63        if not cr:
64            cr = (env.get('LINES', 25), env.get('COLUMNS', 80))
65        return int(cr[1]), int(cr[0])
66
67    def __init__(self, *args):
68        if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__:
69            raise Exception("LLDBTestResult instantiated more than once")
70        super(LLDBTestResult, self).__init__(*args)
71        LLDBTestResult.__singleton__ = self
72        # Now put this singleton into the lldb module namespace.
73        configuration.test_result = self
74        # Computes the format string for displaying the counter.
75        counterWidth = len(str(configuration.suite.countTestCases()))
76        self.fmt = "%" + str(counterWidth) + "d: "
77        self.indentation = ' ' * (counterWidth + 2)
78        # This counts from 1 .. suite.countTestCases().
79        self.counter = 0
80        (width, height) = LLDBTestResult.getTerminalSize()
81        self.results_formatter = configuration.results_formatter_object
82
83    def _config_string(self, test):
84        compiler = getattr(test, "getCompiler", None)
85        arch = getattr(test, "getArchitecture", None)
86        return "%s-%s" % (compiler() if compiler else "",
87                          arch() if arch else "")
88
89    def _exc_info_to_string(self, err, test):
90        """Overrides superclass TestResult's method in order to append
91        our test config info string to the exception info string."""
92        if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"):
93            return '%sConfig=%s-%s' % (super(LLDBTestResult,
94                                             self)._exc_info_to_string(err,
95                                                                       test),
96                                       test.getArchitecture(),
97                                       test.getCompiler())
98        else:
99            return super(LLDBTestResult, self)._exc_info_to_string(err, test)
100
101    def getDescription(self, test):
102        doc_first_line = test.shortDescription()
103        if self.descriptions and doc_first_line:
104            return '\n'.join((str(test), self.indentation + doc_first_line))
105        else:
106            return str(test)
107
108    @staticmethod
109    def _getFileBasedCategories(test):
110        """
111        Returns the list of categories to which this test case belongs by
112        looking for a ".categories" file. We start at the folder the test is in
113        an traverse the hierarchy upwards - we guarantee a .categories to exist
114        at the top level directory so we do not end up looping endlessly.
115        """
116        import inspect
117        import os.path
118        folder = inspect.getfile(test.__class__)
119        folder = os.path.dirname(folder)
120        while folder != '/':
121            categories_file_name = os.path.join(folder, ".categories")
122            if os.path.exists(categories_file_name):
123                categories_file = open(categories_file_name, 'r')
124                categories = categories_file.readline()
125                categories_file.close()
126                categories = str.replace(categories, '\n', '')
127                categories = str.replace(categories, '\r', '')
128                return categories.split(',')
129            else:
130                folder = os.path.dirname(folder)
131                continue
132
133
134    def getCategoriesForTest(self, test):
135        """
136        Gets all the categories for the currently running test method in test case
137        """
138        test_categories = []
139        test_method = getattr(test, test._testMethodName)
140        if test_method is not None and hasattr(test_method, "categories"):
141            test_categories.extend(test_method.categories)
142
143        test_categories.extend(self._getFileBasedCategories(test))
144
145        return test_categories
146
147    def hardMarkAsSkipped(self, test):
148        getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
149        getattr(
150            test,
151            test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
152
153    def checkExclusion(self, exclusion_list, name):
154        if exclusion_list:
155            import re
156            for item in exclusion_list:
157                if re.search(item, name):
158                    return True
159        return False
160
161    def startTest(self, test):
162        if configuration.shouldSkipBecauseOfCategories(
163                self.getCategoriesForTest(test)):
164            self.hardMarkAsSkipped(test)
165        if self.checkExclusion(
166                configuration.skip_tests, test.id()):
167            self.hardMarkAsSkipped(test)
168
169        configuration.setCrashInfoHook(
170            "%s at %s" %
171            (str(test), inspect.getfile(
172                test.__class__)))
173        self.counter += 1
174        # if self.counter == 4:
175        #    import crashinfo
176        #    crashinfo.testCrashReporterDescription(None)
177        test.test_number = self.counter
178        if self.showAll:
179            self.stream.write(self.fmt % self.counter)
180        super(LLDBTestResult, self).startTest(test)
181        if self.results_formatter:
182            self.results_formatter.handle_event(
183                EventBuilder.event_for_start(test))
184
185    def addSuccess(self, test):
186        if self.checkExclusion(
187                configuration.xfail_tests, test.id()):
188            self.addUnexpectedSuccess(test, None)
189            return
190
191        super(LLDBTestResult, self).addSuccess(test)
192        if configuration.parsable:
193            self.stream.write(
194                "PASS: LLDB (%s) :: %s\n" %
195                (self._config_string(test), str(test)))
196        if self.results_formatter:
197            self.results_formatter.handle_event(
198                EventBuilder.event_for_success(test))
199
200    def _isBuildError(self, err_tuple):
201        exception = err_tuple[1]
202        return isinstance(exception, build_exception.BuildError)
203
204    def _getTestPath(self, test):
205        if test is None:
206            return ""
207        elif hasattr(test, "test_filename"):
208            return test.test_filename
209        else:
210            return inspect.getsourcefile(test.__class__)
211
212    def _saveBuildErrorTuple(self, test, err):
213        # Adjust the error description so it prints the build command and build error
214        # rather than an uninformative Python backtrace.
215        build_error = err[1]
216        error_description = "{}\nTest Directory:\n{}".format(
217            str(build_error),
218            os.path.dirname(self._getTestPath(test)))
219        self.errors.append((test, error_description))
220        self._mirrorOutput = True
221
222    def addError(self, test, err):
223        configuration.sdir_has_content = True
224        if self._isBuildError(err):
225            self._saveBuildErrorTuple(test, err)
226        else:
227            super(LLDBTestResult, self).addError(test, err)
228
229        method = getattr(test, "markError", None)
230        if method:
231            method()
232        if configuration.parsable:
233            self.stream.write(
234                "FAIL: LLDB (%s) :: %s\n" %
235                (self._config_string(test), str(test)))
236        if self.results_formatter:
237            # Handle build errors as a separate event type
238            if self._isBuildError(err):
239                error_event = EventBuilder.event_for_build_error(test, err)
240            else:
241                error_event = EventBuilder.event_for_error(test, err)
242            self.results_formatter.handle_event(error_event)
243
244    def addCleanupError(self, test, err):
245        configuration.sdir_has_content = True
246        super(LLDBTestResult, self).addCleanupError(test, err)
247        method = getattr(test, "markCleanupError", None)
248        if method:
249            method()
250        if configuration.parsable:
251            self.stream.write(
252                "CLEANUP ERROR: LLDB (%s) :: %s\n" %
253                (self._config_string(test), str(test)))
254        if self.results_formatter:
255            self.results_formatter.handle_event(
256                EventBuilder.event_for_cleanup_error(
257                    test, err))
258
259    def addFailure(self, test, err):
260        if self.checkExclusion(
261                configuration.xfail_tests, test.id()):
262            self.addExpectedFailure(test, err, None)
263            return
264
265        configuration.sdir_has_content = True
266        super(LLDBTestResult, self).addFailure(test, err)
267        method = getattr(test, "markFailure", None)
268        if method:
269            method()
270        if configuration.parsable:
271            self.stream.write(
272                "FAIL: LLDB (%s) :: %s\n" %
273                (self._config_string(test), str(test)))
274        if configuration.useCategories:
275            test_categories = self.getCategoriesForTest(test)
276            for category in test_categories:
277                if category in configuration.failuresPerCategory:
278                    configuration.failuresPerCategory[
279                        category] = configuration.failuresPerCategory[category] + 1
280                else:
281                    configuration.failuresPerCategory[category] = 1
282        if self.results_formatter:
283            self.results_formatter.handle_event(
284                EventBuilder.event_for_failure(test, err))
285
286    def addExpectedFailure(self, test, err, bugnumber):
287        configuration.sdir_has_content = True
288        super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber)
289        method = getattr(test, "markExpectedFailure", None)
290        if method:
291            method(err, bugnumber)
292        if configuration.parsable:
293            self.stream.write(
294                "XFAIL: LLDB (%s) :: %s\n" %
295                (self._config_string(test), str(test)))
296        if self.results_formatter:
297            self.results_formatter.handle_event(
298                EventBuilder.event_for_expected_failure(
299                    test, err, bugnumber))
300
301    def addSkip(self, test, reason):
302        configuration.sdir_has_content = True
303        super(LLDBTestResult, self).addSkip(test, reason)
304        method = getattr(test, "markSkippedTest", None)
305        if method:
306            method()
307        if configuration.parsable:
308            self.stream.write(
309                "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" %
310                (self._config_string(test), str(test), reason))
311        if self.results_formatter:
312            self.results_formatter.handle_event(
313                EventBuilder.event_for_skip(test, reason))
314
315    def addUnexpectedSuccess(self, test, bugnumber):
316        configuration.sdir_has_content = True
317        super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber)
318        method = getattr(test, "markUnexpectedSuccess", None)
319        if method:
320            method(bugnumber)
321        if configuration.parsable:
322            self.stream.write(
323                "XPASS: LLDB (%s) :: %s\n" %
324                (self._config_string(test), str(test)))
325        if self.results_formatter:
326            self.results_formatter.handle_event(
327                EventBuilder.event_for_unexpected_success(
328                    test, bugnumber))
329