1# DExTer : Debugging Experience Tester
2# ~~~~~~   ~         ~~         ~   ~~
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7"""Base class for subtools that do build/run tests."""
8
9import abc
10from datetime import datetime
11import os
12import sys
13
14from dex.builder import add_builder_tool_arguments
15from dex.builder import handle_builder_tool_options
16from dex.debugger.Debuggers import add_debugger_tool_arguments
17from dex.debugger.Debuggers import handle_debugger_tool_options
18from dex.heuristic.Heuristic import add_heuristic_tool_arguments
19from dex.tools.ToolBase import ToolBase
20from dex.utils import get_root_directory, warn
21from dex.utils.Exceptions import Error, ToolArgumentError
22from dex.utils.ReturnCode import ReturnCode
23
24
25class TestToolBase(ToolBase):
26    def __init__(self, *args, **kwargs):
27        super(TestToolBase, self).__init__(*args, **kwargs)
28        self.build_script: str = None
29
30    def add_tool_arguments(self, parser, defaults):
31        parser.description = self.__doc__
32        add_builder_tool_arguments(parser)
33        add_debugger_tool_arguments(parser, self.context, defaults)
34        add_heuristic_tool_arguments(parser)
35
36        parser.add_argument(
37            'test_path',
38            type=str,
39            metavar='<test-path>',
40            nargs='?',
41            default=os.path.abspath(
42                os.path.join(get_root_directory(), '..', 'tests')),
43            help='directory containing test(s)')
44
45        parser.add_argument(
46            '--results-directory',
47            type=str,
48            metavar='<directory>',
49            default=None,
50            help='directory to save results (default: none)')
51
52    def handle_options(self, defaults):
53        options = self.context.options
54
55        if not options.builder and (options.cflags or options.ldflags):
56            warn(self.context, '--cflags and --ldflags will be ignored when not'
57                               ' using --builder')
58
59        if options.vs_solution:
60            options.vs_solution = os.path.abspath(options.vs_solution)
61            if not os.path.isfile(options.vs_solution):
62                raise Error('<d>could not find VS solution file</> <r>"{}"</>'
63                            .format(options.vs_solution))
64        elif options.binary:
65            options.binary = os.path.abspath(options.binary)
66            if not os.path.isfile(options.binary):
67                raise Error('<d>could not find binary file</> <r>"{}"</>'
68                            .format(options.binary))
69        else:
70            try:
71                self.build_script = handle_builder_tool_options(self.context)
72            except ToolArgumentError as e:
73                raise Error(e)
74
75        try:
76            handle_debugger_tool_options(self.context, defaults)
77        except ToolArgumentError as e:
78            raise Error(e)
79
80        options.test_path = os.path.abspath(options.test_path)
81        options.test_path = os.path.normcase(options.test_path)
82        if not os.path.isfile(options.test_path) and not os.path.isdir(options.test_path):
83            raise Error(
84                '<d>could not find test path</> <r>"{}"</>'.format(
85                    options.test_path))
86
87        if options.results_directory:
88            options.results_directory = os.path.abspath(options.results_directory)
89            if not os.path.isdir(options.results_directory):
90                try:
91                    os.makedirs(options.results_directory, exist_ok=True)
92                except OSError as e:
93                    raise Error(
94                        '<d>could not create directory</> <r>"{}"</> <y>({})</>'.
95                        format(options.results_directory, e.strerror))
96
97    def go(self) -> ReturnCode:  # noqa
98        options = self.context.options
99
100        options.executable = os.path.join(
101            self.context.working_directory.path, 'tmp.exe')
102
103        # Test files contain dexter commands.
104        options.test_files = []
105        # Source files are to be compiled by the builder script and may also
106        # contains dexter commands.
107        options.source_files = []
108        if os.path.isdir(options.test_path):
109            subdirs = sorted([
110                r for r, _, f in os.walk(options.test_path)
111                if 'test.cfg' in f
112            ])
113
114            for subdir in subdirs:
115                for f in os.listdir(subdir):
116                    # TODO: read file extensions from the test.cfg file instead so
117                    # that this isn't just limited to C and C++.
118                    file_path = os.path.normcase(os.path.join(subdir, f))
119                    if f.endswith('.cpp'):
120                        options.source_files.append(file_path)
121                    elif f.endswith('.c'):
122                        options.source_files.append(file_path)
123                    elif f.endswith('.dex'):
124                        options.test_files.append(file_path)
125                # Source files can contain dexter commands too.
126                options.test_files = options.test_files + options.source_files
127
128                self._run_test(self._get_test_name(subdir))
129        else:
130            # We're dealing with a direct file path to a test file. If the file is non
131            # .dex, then it must be a source file.
132            if not options.test_path.endswith('.dex'):
133                options.source_files = [options.test_path]
134            options.test_files = [options.test_path]
135            self._run_test(self._get_test_name(options.test_path))
136
137        return self._handle_results()
138
139    @staticmethod
140    def _is_current_directory(test_directory):
141        return test_directory == '.'
142
143    def _get_test_name(self, test_path):
144        """Get the test name from either the test file, or the sub directory
145        path it's stored in.
146        """
147        # test names are distinguished by their relative path from the
148        # specified test path.
149        test_name = os.path.relpath(test_path,
150                                    self.context.options.test_path)
151        if self._is_current_directory(test_name):
152            test_name = os.path.basename(test_path)
153        return test_name
154
155    @abc.abstractmethod
156    def _run_test(self, test_dir):
157        pass
158
159    @abc.abstractmethod
160    def _handle_results(self) -> ReturnCode:
161        pass
162