1# DExTer : Debugging Experience Tester
2# ~~~~~~   ~         ~~         ~   ~~
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7"""Test tool."""
8
9import math
10import os
11import csv
12import pickle
13import shutil
14
15from dex.builder import run_external_build_script
16from dex.command.ParseCommand import get_command_infos
17from dex.debugger.Debuggers import run_debugger_subprocess
18from dex.debugger.DebuggerControllers.DefaultController import DefaultController
19from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController
20from dex.dextIR.DextIR import DextIR
21from dex.heuristic import Heuristic
22from dex.tools import TestToolBase
23from dex.utils.Exceptions import DebuggerException
24from dex.utils.Exceptions import BuildScriptException, HeuristicException
25from dex.utils.PrettyOutputBase import Stream
26from dex.utils.ReturnCode import ReturnCode
27from dex.dextIR import BuilderIR
28
29
30class TestCase(object):
31    def __init__(self, context, name, heuristic, error):
32        self.context = context
33        self.name = name
34        self.heuristic = heuristic
35        self.error = error
36
37    @property
38    def penalty(self):
39        try:
40            return self.heuristic.penalty
41        except AttributeError:
42            return float('nan')
43
44    @property
45    def max_penalty(self):
46        try:
47            return self.heuristic.max_penalty
48        except AttributeError:
49            return float('nan')
50
51    @property
52    def score(self):
53        try:
54            return self.heuristic.score
55        except AttributeError:
56            return float('nan')
57
58    def __str__(self):
59        if self.error and self.context.options.verbose:
60            verbose_error = str(self.error)
61        else:
62            verbose_error = ''
63
64        if self.error:
65            script_error = (' : {}'.format(
66                self.error.script_error.splitlines()[0]) if getattr(
67                    self.error, 'script_error', None) else '')
68
69            error = ' [{}{}]'.format(
70                str(self.error).splitlines()[0], script_error)
71        else:
72            error = ''
73
74        try:
75            summary = self.heuristic.summary_string
76        except AttributeError:
77            summary = '<r>nan/nan (nan)</>'
78        return '{}: {}{}\n{}'.format(self.name, summary, error, verbose_error)
79
80
81class Tool(TestToolBase):
82    """Run the specified DExTer test(s) with the specified compiler and linker
83    options and produce a dextIR file as well as printing out the debugging
84    experience score calculated by the DExTer heuristic.
85    """
86
87    def __init__(self, *args, **kwargs):
88        super(Tool, self).__init__(*args, **kwargs)
89        self._test_cases = []
90
91    @property
92    def name(self):
93        return 'DExTer test'
94
95    def add_tool_arguments(self, parser, defaults):
96        parser.add_argument('--fail-lt',
97                            type=float,
98                            default=0.0, # By default TEST always succeeds.
99                            help='exit with status FAIL(2) if the test result'
100                                ' is less than this value.',
101                            metavar='<float>')
102        parser.add_argument('--calculate-average',
103                            action="store_true",
104                            help='calculate the average score of every test run')
105        super(Tool, self).add_tool_arguments(parser, defaults)
106
107    def _build_test_case(self):
108        """Build an executable from the test source with the given --builder
109        script and flags (--cflags, --ldflags) in the working directory.
110        Or, if the --binary option has been given, copy the executable provided
111        into the working directory and rename it to match the --builder output
112        or skip if --vs-solution was passed on the command line.
113        """
114
115        if self.context.options.vs_solution:
116            return
117
118        options = self.context.options
119        if options.binary:
120            # Copy user's binary into the tmp working directory
121            shutil.copy(options.binary, options.executable)
122            builderIR = BuilderIR(
123                name='binary',
124                cflags=[options.binary],
125                ldflags='')
126        else:
127            options = self.context.options
128            compiler_options = [options.cflags for _ in options.source_files]
129            linker_options = options.ldflags
130            _, _, builderIR = run_external_build_script(
131                self.context,
132                script_path=self.build_script,
133                source_files=options.source_files,
134                compiler_options=compiler_options,
135                linker_options=linker_options,
136                executable_file=options.executable)
137        return builderIR
138
139    def _init_debugger_controller(self):
140        step_collection = DextIR(
141            executable_path=self.context.options.executable,
142            source_paths=self.context.options.source_files,
143            dexter_version=self.context.version)
144
145        step_collection.commands, new_source_files = get_command_infos(
146            self.context.options.test_files, self.context.options.source_root_dir)
147
148        self.context.options.source_files.extend(list(new_source_files))
149
150        if 'DexLimitSteps' in step_collection.commands:
151            debugger_controller = ConditionalController(self.context, step_collection)
152        else:
153            debugger_controller = DefaultController(self.context, step_collection)
154
155        return debugger_controller
156
157    def _get_steps(self, builderIR):
158        """Generate a list of debugger steps from a test case.
159        """
160        debugger_controller = self._init_debugger_controller()
161        debugger_controller = run_debugger_subprocess(
162            debugger_controller, self.context.working_directory.path)
163        steps = debugger_controller.step_collection
164        steps.builder = builderIR
165        return steps
166
167    def _get_results_basename(self, test_name):
168        def splitall(x):
169            while len(x) > 0:
170              x, y = os.path.split(x)
171              yield y
172        all_components = reversed([x for x in splitall(test_name)])
173        return '_'.join(all_components)
174
175    def _get_results_path(self, test_name):
176        """Returns the path to the test results directory for the test denoted
177        by test_name.
178        """
179        assert self.context.options.results_directory != None
180        return os.path.join(self.context.options.results_directory,
181                            self._get_results_basename(test_name))
182
183    def _get_results_text_path(self, test_name):
184        """Returns path results .txt file for test denoted by test_name.
185        """
186        test_results_path = self._get_results_path(test_name)
187        return '{}.txt'.format(test_results_path)
188
189    def _get_results_pickle_path(self, test_name):
190        """Returns path results .dextIR file for test denoted by test_name.
191        """
192        test_results_path = self._get_results_path(test_name)
193        return '{}.dextIR'.format(test_results_path)
194
195    def _record_steps(self, test_name, steps):
196        """Write out the set of steps out to the test's .txt and .json
197        results file if a results directory has been specified.
198        """
199        if self.context.options.results_directory:
200            output_text_path = self._get_results_text_path(test_name)
201            with open(output_text_path, 'w') as fp:
202                self.context.o.auto(str(steps), stream=Stream(fp))
203
204            output_dextIR_path = self._get_results_pickle_path(test_name)
205            with open(output_dextIR_path, 'wb') as fp:
206                pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
207
208    def _record_score(self, test_name, heuristic):
209        """Write out the test's heuristic score to the results .txt file
210        if a results directory has been specified.
211        """
212        if self.context.options.results_directory:
213            output_text_path = self._get_results_text_path(test_name)
214            with open(output_text_path, 'a') as fp:
215                self.context.o.auto(heuristic.verbose_output, stream=Stream(fp))
216
217    def _record_test_and_display(self, test_case):
218        """Output test case to o stream and record test case internally for
219        handling later.
220        """
221        self.context.o.auto(test_case)
222        self._test_cases.append(test_case)
223
224    def _record_failed_test(self, test_name, exception):
225        """Instantiate a failed test case with failure exception and
226        store internally.
227        """
228        test_case = TestCase(self.context, test_name, None, exception)
229        self._record_test_and_display(test_case)
230
231    def _record_successful_test(self, test_name, steps, heuristic):
232        """Instantiate a successful test run, store test for handling later.
233        Display verbose output for test case if required.
234        """
235        test_case = TestCase(self.context, test_name, heuristic, None)
236        self._record_test_and_display(test_case)
237        if self.context.options.verbose:
238            self.context.o.auto('\n{}\n'.format(steps))
239            self.context.o.auto(heuristic.verbose_output)
240
241    def _run_test(self, test_name):
242        """Attempt to run test files specified in options.source_files. Store
243        result internally in self._test_cases.
244        """
245        try:
246            builderIR = self._build_test_case()
247            steps = self._get_steps(builderIR)
248            self._record_steps(test_name, steps)
249            heuristic_score = Heuristic(self.context, steps)
250            self._record_score(test_name, heuristic_score)
251        except (BuildScriptException, DebuggerException,
252                HeuristicException) as e:
253            self._record_failed_test(test_name, e)
254            return
255
256        self._record_successful_test(test_name, steps, heuristic_score)
257        return
258
259    def _handle_results(self) -> ReturnCode:
260        return_code = ReturnCode.OK
261        options = self.context.options
262
263        if not options.verbose:
264            self.context.o.auto('\n')
265
266        if options.calculate_average:
267            # Calculate and print the average score
268            score_sum = 0.0
269            num_tests = 0
270            for test_case in self._test_cases:
271                score = test_case.score
272                if not test_case.error and not math.isnan(score):
273                    score_sum += test_case.score
274                    num_tests += 1
275
276            if num_tests != 0:
277                print("@avg: ({:.4f})".format(score_sum/num_tests))
278
279        has_failed = lambda test: test.score < options.fail_lt or test.error
280        if any(map(has_failed, self._test_cases)):
281            return_code = ReturnCode.FAIL
282
283        if options.results_directory:
284            summary_path = os.path.join(options.results_directory, 'summary.csv')
285            with open(summary_path, mode='w', newline='') as fp:
286                writer = csv.writer(fp, delimiter=',')
287                writer.writerow(['Test Case', 'Score', 'Error'])
288
289                for test_case in self._test_cases:
290                    writer.writerow([
291                        test_case.name, '{:.4f}'.format(test_case.score),
292                        test_case.error
293                    ])
294
295        return return_code
296