1#!/usr/bin/env python
2
3"""A script to generate FileCheck statements for 'opt' regression tests.
4
5This script is a utility to update LLVM opt test cases with new
6FileCheck patterns. It can either update all of the tests in the file or
7a single test function.
8
9Example usage:
10$ update_test_checks.py --opt=../bin/opt test/foo.ll
11
12Workflow:
131. Make a compiler patch that requires updating some number of FileCheck lines
14   in regression test files.
152. Save the patch and revert it from your local work area.
163. Update the RUN-lines in the affected regression tests to look canonical.
17   Example: "; RUN: opt < %s -instcombine -S | FileCheck %s"
184. Refresh the FileCheck lines for either the entire file or select functions by
19   running this script.
205. Commit the fresh baseline of checks.
216. Apply your patch from step 1 and rebuild your local binaries.
227. Re-run this script on affected regression tests.
238. Check the diffs to ensure the script has done something reasonable.
249. Submit a patch including the regression test diffs for review.
25
26A common pattern is to have the script insert complete checking of every
27instruction. Then, edit it down to only check the relevant instructions.
28The script is designed to make adding checks to a test case fast, it is *not*
29designed to be authoratitive about what constitutes a good test!
30"""
31
32from __future__ import print_function
33
34import argparse
35import itertools
36import os         # Used to advertise this file's name ("autogenerated_note").
37import string
38import subprocess
39import sys
40import tempfile
41import re
42
43from UpdateTestChecks import common
44
45ADVERT = '; NOTE: Assertions have been autogenerated by '
46
47# RegEx: this is where the magic happens.
48
49IR_FUNCTION_RE = re.compile('^\s*define\s+(?:internal\s+)?[^@]*@([\w-]+)\s*\(')
50
51
52
53
54
55def main():
56  from argparse import RawTextHelpFormatter
57  parser = argparse.ArgumentParser(description=__doc__, formatter_class=RawTextHelpFormatter)
58  parser.add_argument('-v', '--verbose', action='store_true',
59                      help='Show verbose output')
60  parser.add_argument('--opt-binary', default='opt',
61                      help='The opt binary used to generate the test case')
62  parser.add_argument(
63      '--function', help='The function in the test file to update')
64  parser.add_argument('tests', nargs='+')
65  args = parser.parse_args()
66
67  autogenerated_note = (ADVERT + 'utils/' + os.path.basename(__file__))
68
69  opt_basename = os.path.basename(args.opt_binary)
70  if (opt_basename != "opt"):
71    print('ERROR: Unexpected opt name: ' + opt_basename, file=sys.stderr)
72    sys.exit(1)
73
74  for test in args.tests:
75    if args.verbose:
76      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
77    with open(test) as f:
78      input_lines = [l.rstrip() for l in f]
79
80    raw_lines = [m.group(1)
81                 for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
82    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
83    for l in raw_lines[1:]:
84      if run_lines[-1].endswith("\\"):
85        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
86      else:
87        run_lines.append(l)
88
89    if args.verbose:
90      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
91      for l in run_lines:
92        print('  RUN: ' + l, file=sys.stderr)
93
94    prefix_list = []
95    for l in run_lines:
96      (tool_cmd, filecheck_cmd) = tuple([cmd.strip() for cmd in l.split('|', 1)])
97
98      if not tool_cmd.startswith(opt_basename + ' '):
99        print('WARNING: Skipping non-%s RUN line: %s' % (opt_basename, l), file=sys.stderr)
100        continue
101
102      if not filecheck_cmd.startswith('FileCheck '):
103        print('WARNING: Skipping non-FileChecked RUN line: ' + l, file=sys.stderr)
104        continue
105
106      tool_cmd_args = tool_cmd[len(opt_basename):].strip()
107      tool_cmd_args = tool_cmd_args.replace('< %s', '').replace('%s', '').strip()
108
109      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
110                               for item in m.group(1).split(',')]
111      if not check_prefixes:
112        check_prefixes = ['CHECK']
113
114      # FIXME: We should use multiple check prefixes to common check lines. For
115      # now, we just ignore all but the last.
116      prefix_list.append((check_prefixes, tool_cmd_args))
117
118    func_dict = {}
119    for prefixes, _ in prefix_list:
120      for prefix in prefixes:
121        func_dict.update({prefix: dict()})
122    for prefixes, opt_args in prefix_list:
123      if args.verbose:
124        print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
125        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
126
127      raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
128      common.build_function_body_dictionary(
129              common.OPT_FUNCTION_RE, common.scrub_body, [],
130              raw_tool_output, prefixes, func_dict, args.verbose)
131
132    is_in_function = False
133    is_in_function_start = False
134    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
135    if args.verbose:
136      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
137    output_lines = []
138    output_lines.append(autogenerated_note)
139
140    for input_line in input_lines:
141      if is_in_function_start:
142        if input_line == '':
143          continue
144        if input_line.lstrip().startswith(';'):
145          m = common.CHECK_RE.match(input_line)
146          if not m or m.group(1) not in prefix_set:
147            output_lines.append(input_line)
148            continue
149
150        # Print out the various check lines here.
151        common.add_ir_checks(output_lines, ';', prefix_list, func_dict, func_name)
152        is_in_function_start = False
153
154      if is_in_function:
155        if common.should_add_line_to_output(input_line, prefix_set):
156          # This input line of the function body will go as-is into the output.
157          # Except make leading whitespace uniform: 2 spaces.
158          input_line = common.SCRUB_LEADING_WHITESPACE_RE.sub(r'  ', input_line)
159          output_lines.append(input_line)
160        else:
161          continue
162        if input_line.strip() == '}':
163          is_in_function = False
164        continue
165
166      # Discard any previous script advertising.
167      if input_line.startswith(ADVERT):
168        continue
169
170      # If it's outside a function, it just gets copied to the output.
171      output_lines.append(input_line)
172
173      m = IR_FUNCTION_RE.match(input_line)
174      if not m:
175        continue
176      func_name = m.group(1)
177      if args.function is not None and func_name != args.function:
178        # When filtering on a specific function, skip all others.
179        continue
180      is_in_function = is_in_function_start = True
181
182    if args.verbose:
183      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
184
185    with open(test, 'wb') as f:
186      f.writelines([l + '\n' for l in output_lines])
187
188
189if __name__ == '__main__':
190  main()
191