1# DExTer : Debugging Experience Tester
2# ~~~~~~   ~         ~~         ~   ~~
3#
4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5# See https://llvm.org/LICENSE.txt for license information.
6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7"""Parse a DExTer command. In particular, ensure that only a very limited
8subset of Python is allowed, in order to prevent the possibility of unsafe
9Python code being embedded within DExTer commands.
10"""
11
12import os
13import unittest
14from copy import copy
15from pathlib import PurePath
16from collections import defaultdict, OrderedDict
17
18from dex.utils.Exceptions import CommandParseError
19
20from dex.command.CommandBase import CommandBase
21from dex.command.commands.DexCommandLine import DexCommandLine
22from dex.command.commands.DexDeclareFile import DexDeclareFile
23from dex.command.commands.DexDeclareAddress import DexDeclareAddress
24from dex.command.commands.DexExpectProgramState import DexExpectProgramState
25from dex.command.commands.DexExpectStepKind import DexExpectStepKind
26from dex.command.commands.DexExpectStepOrder import DexExpectStepOrder
27from dex.command.commands.DexExpectWatchType import DexExpectWatchType
28from dex.command.commands.DexExpectWatchValue import DexExpectWatchValue
29from dex.command.commands.DexExpectWatchBase import AddressExpression, DexExpectWatchBase
30from dex.command.commands.DexLabel import DexLabel
31from dex.command.commands.DexLimitSteps import DexLimitSteps
32from dex.command.commands.DexFinishTest import DexFinishTest
33from dex.command.commands.DexUnreachable import DexUnreachable
34from dex.command.commands.DexWatch import DexWatch
35from dex.utils import Timer
36from dex.utils.Exceptions import CommandParseError, DebuggerException
37
38def _get_valid_commands():
39    """Return all top level DExTer test commands.
40
41    Returns:
42        { name (str): command (class) }
43    """
44    return {
45      DexCommandLine.get_name() : DexCommandLine,
46      DexDeclareAddress.get_name() : DexDeclareAddress,
47      DexDeclareFile.get_name() : DexDeclareFile,
48      DexExpectProgramState.get_name() : DexExpectProgramState,
49      DexExpectStepKind.get_name() : DexExpectStepKind,
50      DexExpectStepOrder.get_name() : DexExpectStepOrder,
51      DexExpectWatchType.get_name() : DexExpectWatchType,
52      DexExpectWatchValue.get_name() : DexExpectWatchValue,
53      DexLabel.get_name() : DexLabel,
54      DexLimitSteps.get_name() : DexLimitSteps,
55      DexFinishTest.get_name() : DexFinishTest,
56      DexUnreachable.get_name() : DexUnreachable,
57      DexWatch.get_name() : DexWatch
58    }
59
60
61def _get_command_name(command_raw: str) -> str:
62    """Return command name by splitting up DExTer command contained in
63    command_raw on the first opening paranthesis and further stripping
64    any potential leading or trailing whitespace.
65    """
66    return command_raw.split('(', 1)[0].rstrip()
67
68
69def _merge_subcommands(command_name: str, valid_commands: dict) -> dict:
70    """Merge valid_commands and command_name's subcommands into a new dict.
71
72    Returns:
73        { name (str): command (class) }
74    """
75    subcommands = valid_commands[command_name].get_subcommands()
76    if subcommands:
77        return { **valid_commands, **subcommands }
78    return valid_commands
79
80
81def _build_command(command_type, labels, addresses, raw_text: str, path: str, lineno: str) -> CommandBase:
82    """Build a command object from raw text.
83
84    This function will call eval().
85
86    Raises:
87        Any exception that eval() can raise.
88
89    Returns:
90        A dexter command object.
91    """
92    def label_to_line(label_name: str) -> int:
93        line = labels.get(label_name, None)
94        if line != None:
95            return line
96        raise format_unresolved_label_err(label_name, raw_text, path, lineno)
97
98    def get_address_object(address_name: str, offset: int=0):
99        if address_name not in addresses:
100            raise format_undeclared_address_err(address_name, raw_text, path, lineno)
101        return AddressExpression(address_name, offset)
102
103    valid_commands = _merge_subcommands(
104        command_type.get_name(), {
105            'ref': label_to_line,
106            'address': get_address_object,
107            command_type.get_name(): command_type,
108        })
109
110    # pylint: disable=eval-used
111    command = eval(raw_text, valid_commands)
112    # pylint: enable=eval-used
113    command.raw_text = raw_text
114    command.path = path
115    command.lineno = lineno
116    return command
117
118
119def _search_line_for_cmd_start(line: str, start: int, valid_commands: dict) -> int:
120    """Scan `line` for a string matching any key in `valid_commands`.
121
122    Start searching from `start`.
123    Commands escaped with `\` (E.g. `\DexLabel('a')`) are ignored.
124
125    Returns:
126        int: the index of the first character of the matching string in `line`
127        or -1 if no command is found.
128    """
129    for command in valid_commands:
130        idx = line.find(command, start)
131        if idx != -1:
132            # Ignore escaped '\' commands.
133            if idx > 0 and line[idx - 1] == '\\':
134                continue
135            return idx
136    return -1
137
138
139def _search_line_for_cmd_end(line: str, start: int, paren_balance: int) -> (int, int):
140    """Find the end of a command by looking for balanced parentheses.
141
142    Args:
143        line: String to scan.
144        start: Index into `line` to start looking.
145        paren_balance(int): paren_balance after previous call.
146
147    Note:
148        On the first call `start` should point at the opening parenthesis and
149        `paren_balance` should be set to 0. Subsequent calls should pass in the
150        returned `paren_balance`.
151
152    Returns:
153        ( end,  paren_balance )
154        Where end is 1 + the index of the last char in the command or, if the
155        parentheses are not balanced, the end of the line.
156
157        paren_balance will be 0 when the parentheses are balanced.
158    """
159    for end in range(start, len(line)):
160        ch = line[end]
161        if ch == '(':
162            paren_balance += 1
163        elif ch == ')':
164            paren_balance -=1
165        if paren_balance == 0:
166            break
167    end += 1
168    return (end, paren_balance)
169
170
171class TextPoint():
172    def __init__(self, line, char):
173        self.line = line
174        self.char = char
175
176    def get_lineno(self):
177        return self.line + 1
178
179    def get_column(self):
180        return self.char + 1
181
182
183def format_unresolved_label_err(label: str, src: str, filename: str, lineno) -> CommandParseError:
184    err = CommandParseError()
185    err.src = src
186    err.caret = '' # Don't bother trying to point to the bad label.
187    err.filename = filename
188    err.lineno = lineno
189    err.info = f'Unresolved label: \'{label}\''
190    return err
191
192def format_undeclared_address_err(address: str, src: str, filename: str, lineno) -> CommandParseError:
193    err = CommandParseError()
194    err.src = src
195    err.caret = '' # Don't bother trying to point to the bad address.
196    err.filename = filename
197    err.lineno = lineno
198    err.info = f'Undeclared address: \'{address}\''
199    return err
200
201def format_parse_err(msg: str, path: str, lines: list, point: TextPoint) -> CommandParseError:
202    err = CommandParseError()
203    err.filename = path
204    err.src = lines[point.line].rstrip()
205    err.lineno = point.get_lineno()
206    err.info = msg
207    err.caret = '{}<r>^</>'.format(' ' * (point.char))
208    return err
209
210
211def skip_horizontal_whitespace(line, point):
212    for idx, char in enumerate(line[point.char:]):
213        if char not in ' \t':
214            point.char += idx
215            return
216
217
218def add_line_label(labels, label, cmd_path, cmd_lineno):
219    # Enforce unique line labels.
220    if label.eval() in labels:
221        err = CommandParseError()
222        err.info = f'Found duplicate line label: \'{label.eval()}\''
223        err.lineno = cmd_lineno
224        err.filename = cmd_path
225        err.src = label.raw_text
226        # Don't both trying to point to it since we're only printing the raw
227        # command, which isn't much text.
228        err.caret = ''
229        raise err
230    labels[label.eval()] = label.get_line()
231
232def add_address(addresses, address, cmd_path, cmd_lineno):
233    # Enforce unique address variables.
234    address_name = address.get_address_name()
235    if address_name in addresses:
236        err = CommandParseError()
237        err.info = f'Found duplicate address: \'{address_name}\''
238        err.lineno = cmd_lineno
239        err.filename = cmd_path
240        err.src = address.raw_text
241        # Don't both trying to point to it since we're only printing the raw
242        # command, which isn't much text.
243        err.caret = ''
244        raise err
245    addresses.append(address_name)
246
247def _find_all_commands_in_file(path, file_lines, valid_commands, source_root_dir):
248    labels = {} # dict of {name: line}.
249    addresses = [] # list of addresses.
250    address_resolutions = {}
251    cmd_path = path
252    declared_files = set()
253    commands = defaultdict(dict)
254    paren_balance = 0
255    region_start = TextPoint(0, 0)
256
257    for region_start.line in range(len(file_lines)):
258        line = file_lines[region_start.line]
259        region_start.char = 0
260
261        # Search this line till we find no more commands.
262        while True:
263            # If parens are currently balanced we can look for a new command.
264            if paren_balance == 0:
265                region_start.char = _search_line_for_cmd_start(line, region_start.char, valid_commands)
266                if region_start.char == -1:
267                    break # Read next line.
268
269                command_name = _get_command_name(line[region_start.char:])
270                cmd_point = copy(region_start)
271                cmd_text_list = [command_name]
272
273                region_start.char += len(command_name) # Start searching for parens after cmd.
274                skip_horizontal_whitespace(line, region_start)
275                if region_start.char >= len(line) or line[region_start.char] != '(':
276                    raise format_parse_err(
277                        "Missing open parenthesis", path, file_lines, region_start)
278
279            end, paren_balance = _search_line_for_cmd_end(line, region_start.char, paren_balance)
280            # Add this text blob to the command.
281            cmd_text_list.append(line[region_start.char:end])
282            # Move parse ptr to end of line or parens.
283            region_start.char = end
284
285            # If the parens are unbalanced start reading the next line in an attempt
286            # to find the end of the command.
287            if paren_balance != 0:
288                break  # Read next line.
289
290            # Parens are balanced, we have a full command to evaluate.
291            raw_text = "".join(cmd_text_list)
292            try:
293                command = _build_command(
294                    valid_commands[command_name],
295                    labels,
296                    addresses,
297                    raw_text,
298                    cmd_path,
299                    cmd_point.get_lineno(),
300                )
301            except SyntaxError as e:
302                # This err should point to the problem line.
303                err_point = copy(cmd_point)
304                # To e the command start is the absolute start, so use as offset.
305                err_point.line += e.lineno - 1 # e.lineno is a position, not index.
306                err_point.char += e.offset - 1 # e.offset is a position, not index.
307                raise format_parse_err(e.msg, path, file_lines, err_point)
308            except TypeError as e:
309                # This err should always point to the end of the command name.
310                err_point = copy(cmd_point)
311                err_point.char += len(command_name)
312                raise format_parse_err(str(e), path, file_lines, err_point)
313            else:
314                if type(command) is DexLabel:
315                    add_line_label(labels, command, path, cmd_point.get_lineno())
316                elif type(command) is DexDeclareAddress:
317                    add_address(addresses, command, path, cmd_point.get_lineno())
318                elif type(command) is DexDeclareFile:
319                    cmd_path = command.declared_file
320                    if not os.path.isabs(cmd_path):
321                        source_dir = (source_root_dir if source_root_dir else
322                                      os.path.dirname(path))
323                        cmd_path = os.path.join(source_dir, cmd_path)
324                    # TODO: keep stored paths as PurePaths for 'longer'.
325                    cmd_path = str(PurePath(cmd_path))
326                    declared_files.add(cmd_path)
327                elif type(command) is DexCommandLine and 'DexCommandLine' in commands:
328                    msg = "More than one DexCommandLine in file"
329                    raise format_parse_err(msg, path, file_lines, err_point)
330
331                assert (path, cmd_point) not in commands[command_name], (
332                    command_name, commands[command_name])
333                commands[command_name][path, cmd_point] = command
334
335    if paren_balance != 0:
336        # This err should always point to the end of the command name.
337        err_point = copy(cmd_point)
338        err_point.char += len(command_name)
339        msg = "Unbalanced parenthesis starting here"
340        raise format_parse_err(msg, path, file_lines, err_point)
341    return dict(commands), declared_files
342
343def _find_all_commands(test_files, source_root_dir):
344    commands = defaultdict(dict)
345    valid_commands = _get_valid_commands()
346    new_source_files = set()
347    for test_file in test_files:
348        with open(test_file) as fp:
349            lines = fp.readlines()
350        file_commands, declared_files = _find_all_commands_in_file(
351            test_file, lines, valid_commands, source_root_dir)
352        for command_name in file_commands:
353            commands[command_name].update(file_commands[command_name])
354        new_source_files |= declared_files
355
356    return dict(commands), new_source_files
357
358def get_command_infos(test_files, source_root_dir):
359  with Timer('parsing commands'):
360      try:
361          commands, new_source_files = _find_all_commands(test_files, source_root_dir)
362          command_infos = OrderedDict()
363          for command_type in commands:
364              for command in commands[command_type].values():
365                  if command_type not in command_infos:
366                      command_infos[command_type] = []
367                  command_infos[command_type].append(command)
368          return OrderedDict(command_infos), new_source_files
369      except CommandParseError as e:
370          msg = 'parser error: <d>{}({}):</> {}\n{}\n{}\n'.format(
371                e.filename, e.lineno, e.info, e.src, e.caret)
372          raise DebuggerException(msg)
373
374class TestParseCommand(unittest.TestCase):
375    class MockCmd(CommandBase):
376        """A mock DExTer command for testing parsing.
377
378        Args:
379            value (str): Unique name for this instance.
380        """
381
382        def __init__(self, *args):
383           self.value = args[0]
384
385        def get_name():
386            return __class__.__name__
387
388        def eval(this):
389            pass
390
391
392    def __init__(self, *args):
393        super().__init__(*args)
394
395        self.valid_commands = {
396            TestParseCommand.MockCmd.get_name() : TestParseCommand.MockCmd
397        }
398
399
400    def _find_all_commands_in_lines(self, lines):
401        """Use DExTer parsing methods to find all the mock commands in lines.
402
403        Returns:
404            { cmd_name: { (path, line): command_obj } }
405        """
406        cmds, declared_files = _find_all_commands_in_file(__file__, lines, self.valid_commands, None)
407        return cmds
408
409
410    def _find_all_mock_values_in_lines(self, lines):
411        """Use DExTer parsing methods to find all mock command values in lines.
412
413        Returns:
414            values (list(str)): MockCmd values found in lines.
415        """
416        cmds = self._find_all_commands_in_lines(lines)
417        mocks = cmds.get(TestParseCommand.MockCmd.get_name(), None)
418        return [v.value for v in mocks.values()] if mocks else []
419
420
421    def test_parse_inline(self):
422        """Commands can be embedded in other text."""
423
424        lines = [
425            'MockCmd("START") Lorem ipsum dolor sit amet, consectetur\n',
426            'adipiscing elit, MockCmd("EMBEDDED") sed doeiusmod tempor,\n',
427            'incididunt ut labore et dolore magna aliqua.\n'
428        ]
429
430        values = self._find_all_mock_values_in_lines(lines)
431
432        self.assertTrue('START' in values)
433        self.assertTrue('EMBEDDED' in values)
434
435
436    def test_parse_multi_line_comment(self):
437        """Multi-line commands can embed comments."""
438
439        lines = [
440            'Lorem ipsum dolor sit amet, consectetur\n',
441            'adipiscing elit, sed doeiusmod tempor,\n',
442            'incididunt ut labore et MockCmd(\n',
443            '    "WITH_COMMENT" # THIS IS A COMMENT\n',
444            ') dolore magna aliqua. Ut enim ad minim\n',
445        ]
446
447        values = self._find_all_mock_values_in_lines(lines)
448
449        self.assertTrue('WITH_COMMENT' in values)
450
451    def test_parse_empty(self):
452        """Empty files are silently ignored."""
453
454        lines = []
455        values = self._find_all_mock_values_in_lines(lines)
456        self.assertTrue(len(values) == 0)
457
458    def test_parse_bad_whitespace(self):
459        """Throw exception when parsing badly formed whitespace."""
460        lines = [
461            'MockCmd\n',
462            '("XFAIL_CMD_LF_PAREN")\n',
463        ]
464
465        with self.assertRaises(CommandParseError):
466            values = self._find_all_mock_values_in_lines(lines)
467
468    def test_parse_good_whitespace(self):
469        """Try to emulate python whitespace rules"""
470
471        lines = [
472            'MockCmd("NONE")\n',
473            'MockCmd    ("SPACE")\n',
474            'MockCmd\t\t("TABS")\n',
475            'MockCmd(    "ARG_SPACE"    )\n',
476            'MockCmd(\t\t"ARG_TABS"\t\t)\n',
477            'MockCmd(\n',
478            '"CMD_PAREN_LF")\n',
479        ]
480
481        values = self._find_all_mock_values_in_lines(lines)
482
483        self.assertTrue('NONE' in values)
484        self.assertTrue('SPACE' in values)
485        self.assertTrue('TABS' in values)
486        self.assertTrue('ARG_SPACE' in values)
487        self.assertTrue('ARG_TABS' in values)
488        self.assertTrue('CMD_PAREN_LF' in values)
489
490
491    def test_parse_share_line(self):
492        """More than one command can appear on one line."""
493
494        lines = [
495            'MockCmd("START") MockCmd("CONSECUTIVE") words '
496                'MockCmd("EMBEDDED") more words\n'
497        ]
498
499        values = self._find_all_mock_values_in_lines(lines)
500
501        self.assertTrue('START' in values)
502        self.assertTrue('CONSECUTIVE' in values)
503        self.assertTrue('EMBEDDED' in values)
504
505
506    def test_parse_escaped(self):
507        """Escaped commands are ignored."""
508
509        lines = [
510            'words \MockCmd("IGNORED") words words words\n'
511        ]
512
513        values = self._find_all_mock_values_in_lines(lines)
514
515        self.assertFalse('IGNORED' in values)
516